HMVMXR0.cpp@ 45481

Last change on this file since 45481 was 45481, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: More assertions.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 369.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45481 2013-04-11 11:09:28Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
206	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
207	#endif
208	#if 0
209	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
210	unsigned rcReason);
211	#endif
212
213	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264
265	/*******************************************************************************
266	* Global Variables *
267	*******************************************************************************/
268	/** @todo Move this to hm_vmx.h. */
269	/**
270	* VM-exit handler.
271	*
272	* @returns VBox status code.
273	* @param pVM Pointer to the VM.
274	* @param pVCpu Pointer to the VMCPU.
275	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
276	* out-of-sync. Make sure to update the required
277	* fields before using them.
278	* @param pVmxTransient Pointer to the VMX-transient structure.
279	*/
280	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	/** Pointer to VM-exit handler. */
282	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
283
284	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
285	{
286	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
287	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
288	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
289	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
290	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
291	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
292	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
293	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
294	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
295	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
296	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
297	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
298	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
299	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
300	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
301	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
302	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
303	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
304	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
305	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
306	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
307	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
308	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
309	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
310	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
311	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
312	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
313	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
314	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
315	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
316	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
317	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
318	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
319	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
320	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
321	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
322	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
323	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
324	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
326	/* 40 UNDEFINED */ hmR0VmxExitPause,
327	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
328	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
329	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
330	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
331	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
333	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
334	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
335	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
336	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
337	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
338	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
339	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
340	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
341	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
342	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
343	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
344	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
345	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
346	};
347
348	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
349	{
350	/* 0 */ "(Not Used)",
351	/* 1 */ "VMCALL executed in VMX root operation.",
352	/* 2 */ "VMCLEAR with invalid physical address.",
353	/* 3 */ "VMCLEAR with VMXON pointer.",
354	/* 4 */ "VMLAUNCH with non-clear VMCS.",
355	/* 5 */ "VMRESUME with non-launched VMCS.",
356	/* 6 */ "VMRESUME after VMXOFF",
357	/* 7 */ "VM entry with invalid control fields.",
358	/* 8 */ "VM entry with invalid host state fields.",
359	/* 9 */ "VMPTRLD with invalid physical address.",
360	/* 10 */ "VMPTRLD with VMXON pointer.",
361	/* 11 */ "VMPTRLD with incorrect revision identifier.",
362	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
363	/* 13 */ "VMWRITE to read-only VMCS component.",
364	/* 14 */ "(Not Used)",
365	/* 15 */ "VMXON executed in VMX root operation.",
366	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
367	/* 17 */ "VM entry with non-launched executing VMCS.",
368	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
369	/* 19 */ "VMCALL with non-clear VMCS.",
370	/* 20 */ "VMCALL with invalid VM-exit control fields.",
371	/* 21 */ "(Not Used)",
372	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
373	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
374	/* 24 */ "VMCALL with invalid SMM-monitor features.",
375	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
376	/* 26 */ "VM entry with events blocked by MOV SS.",
377	/* 27 */ "(Not Used)",
378	/* 28 */ "Invalid operand to INVEPT/INVVPID."
379	};
380
381
382	/**
383	* Updates the VM's last error record. If there was a VMX instruction error,
384	* reads the error data from the VMCS and updates VCPU's last error record as
385	* well.
386	*
387	* @param pVM Pointer to the VM.
388	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
389	* VERR_VMX_UNABLE_TO_START_VM or
390	* VERR_VMX_INVALID_VMCS_FIELD).
391	* @param rc The error code.
392	*/
393	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
394	{
395	AssertPtr(pVM);
396	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
397	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
398	{
399	AssertPtrReturnVoid(pVCpu);
400	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
401	}
402	pVM->hm.s.lLastError = rc;
403	}
404
405
406	/**
407	* Reads the VM-entry interruption-information field from the VMCS into the VMX
408	* transient structure.
409	*
410	* @returns VBox status code.
411	* @param pVmxTransient Pointer to the VMX transient structure.
412	*
413	* @remarks No-long-jump zone!!!
414	*/
415	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
416	{
417	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
418	AssertRCReturn(rc, rc);
419	return VINF_SUCCESS;
420	}
421
422
423	/**
424	* Reads the VM-entry exception error code field from the VMCS into
425	* the VMX transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVCpu Pointer to the VMCPU.
446	* @param pVmxTransient Pointer to the VMX transient structure.
447	*
448	* @remarks No-long-jump zone!!!
449	*/
450	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
451	{
452	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
453	AssertRCReturn(rc, rc);
454	return VINF_SUCCESS;
455	}
456
457
458	/**
459	* Reads the VM-exit interruption-information field from the VMCS into the VMX
460	* transient structure.
461	*
462	* @returns VBox status code.
463	* @param pVCpu Pointer to the VMCPU.
464	* @param pVmxTransient Pointer to the VMX transient structure.
465	*/
466	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
467	{
468	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
469	{
470	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
471	AssertRCReturn(rc, rc);
472	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
473	}
474	return VINF_SUCCESS;
475	}
476
477
478	/**
479	* Reads the VM-exit interruption error code from the VMCS into the VMX
480	* transient structure.
481	*
482	* @returns VBox status code.
483	* @param pVCpu Pointer to the VMCPU.
484	* @param pVmxTransient Pointer to the VMX transient structure.
485	*/
486	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
487	{
488	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
489	{
490	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
491	AssertRCReturn(rc, rc);
492	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
493	}
494	return VINF_SUCCESS;
495	}
496
497
498	/**
499	* Reads the VM-exit instruction length field from the VMCS into the VMX
500	* transient structure.
501	*
502	* @returns VBox status code.
503	* @param pVCpu Pointer to the VMCPU.
504	* @param pVmxTransient Pointer to the VMX transient structure.
505	*/
506	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
507	{
508	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
509	{
510	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
511	AssertRCReturn(rc, rc);
512	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
513	}
514	return VINF_SUCCESS;
515	}
516
517
518	/**
519	* Reads the exit qualification from the VMCS into the VMX transient structure.
520	*
521	* @returns VBox status code.
522	* @param pVCpu Pointer to the VMCPU.
523	* @param pVmxTransient Pointer to the VMX transient structure.
524	*/
525	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
526	{
527	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
528	{
529	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
530	AssertRCReturn(rc, rc);
531	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
532	}
533	return VINF_SUCCESS;
534	}
535
536
537	/**
538	* Reads the IDT-vectoring information field from the VMCS into the VMX
539	* transient structure.
540	*
541	* @returns VBox status code.
542	* @param pVmxTransient Pointer to the VMX transient structure.
543	*
544	* @remarks No-long-jump zone!!!
545	*/
546	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
547	{
548	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
549	{
550	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
551	AssertRCReturn(rc, rc);
552	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
553	}
554	return VINF_SUCCESS;
555	}
556
557
558	/**
559	* Reads the IDT-vectoring error code from the VMCS into the VMX
560	* transient structure.
561	*
562	* @returns VBox status code.
563	* @param pVmxTransient Pointer to the VMX transient structure.
564	*/
565	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
566	{
567	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
568	{
569	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
570	AssertRCReturn(rc, rc);
571	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
572	}
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Enters VMX root mode operation on the current CPU.
579	*
580	* @returns VBox status code.
581	* @param pVM Pointer to the VM (optional, can be NULL, after
582	* a resume).
583	* @param HCPhysCpuPage Physical address of the VMXON region.
584	* @param pvCpuPage Pointer to the VMXON region.
585	*/
586	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
587	{
588	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
589	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
590
591	if (pVM)
592	{
593	/* Write the VMCS revision dword to the VMXON region. */
594	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
595	}
596
597	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
598	RTCCUINTREG fFlags = ASMIntDisableFlags();
599
600	/* Enable the VMX bit in CR4 if necessary. */
601	RTCCUINTREG uCr4 = ASMGetCR4();
602	if (!(uCr4 & X86_CR4_VMXE))
603	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
604
605	/* Enter VMX root mode. */
606	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
607	if (RT_FAILURE(rc))
608	ASMSetCR4(uCr4);
609
610	/* Restore interrupts. */
611	ASMSetFlags(fFlags);
612	return rc;
613	}
614
615
616	/**
617	* Exits VMX root mode operation on the current CPU.
618	*
619	* @returns VBox status code.
620	*/
621	static int hmR0VmxLeaveRootMode(void)
622	{
623	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
624	RTCCUINTREG fFlags = ASMIntDisableFlags();
625	int rc = VINF_SUCCESS;
626
627	/* If we're for some reason not in VMX root mode, then don't leave it. */
628	if (ASMGetCR4() & X86_CR4_VMXE)
629	{
630	/* Exit VMX root mode and clear the VMX bit in CR4 */
631	VMXDisable();
632	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
633	}
634	else
635	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Allocates and maps one physically contiguous page. The allocated page is
645	* zero'd out. (Used by various VT-x structures).
646	*
647	* @returns IPRT status code.
648	* @param pMemObj Pointer to the ring-0 memory object.
649	* @param ppVirt Where to store the virtual address of the
650	* allocation.
651	* @param pPhys Where to store the physical address of the
652	* allocation.
653	*/
654	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
655	{
656	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
657	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
658	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
659
660	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
661	if (RT_FAILURE(rc))
662	return rc;
663	ppVirt = RTR0MemObjAddress(pMemObj);
664	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
665	ASMMemZero32(*ppVirt, PAGE_SIZE);
666	return VINF_SUCCESS;
667	}
668
669
670	/**
671	* Frees and unmaps an allocated physical page.
672	*
673	* @param pMemObj Pointer to the ring-0 memory object.
674	* @param ppVirt Where to re-initialize the virtual address of
675	* allocation as 0.
676	* @param pHCPhys Where to re-initialize the physical address of the
677	* allocation as 0.
678	*/
679	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
680	{
681	AssertPtr(pMemObj);
682	AssertPtr(ppVirt);
683	AssertPtr(pHCPhys);
684	if (*pMemObj != NIL_RTR0MEMOBJ)
685	{
686	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
687	AssertRC(rc);
688	*pMemObj = NIL_RTR0MEMOBJ;
689	*ppVirt = 0;
690	*pHCPhys = 0;
691	}
692	}
693
694
695	/**
696	* Worker function to free VT-x related structures.
697	*
698	* @returns IPRT status code.
699	* @param pVM Pointer to the VM.
700	*/
701	static void hmR0VmxStructsFree(PVM pVM)
702	{
703	for (VMCPUID i = 0; i < pVM->cCpus; i++)
704	{
705	PVMCPU pVCpu = &pVM->aCpus[i];
706	AssertPtr(pVCpu);
707
708	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
709	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
710
711	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
712	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
713
714	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
715	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
716	}
717
718	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
719	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
720	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
721	#endif
722	}
723
724
725	/**
726	* Worker function to allocate VT-x related VM structures.
727	*
728	* @returns IPRT status code.
729	* @param pVM Pointer to the VM.
730	*/
731	static int hmR0VmxStructsAlloc(PVM pVM)
732	{
733	/*
734	* Initialize members up-front so we can cleanup properly on allocation failure.
735	*/
736	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
737	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
738	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
739	pVM->hm.s.vmx.HCPhys##a_Name = 0;
740
741	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
742	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
743	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
744	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
745
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
748	#endif
749	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
750
751	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
752	for (VMCPUID i = 0; i < pVM->cCpus; i++)
753	{
754	PVMCPU pVCpu = &pVM->aCpus[i];
755	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
756	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
757	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
758	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
759	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
760	}
761	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
762	#undef VMXLOCAL_INIT_VM_MEMOBJ
763
764	/*
765	* Allocate all the VT-x structures.
766	*/
767	int rc = VINF_SUCCESS;
768	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
769	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
770	if (RT_FAILURE(rc))
771	goto cleanup;
772	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
773	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
774	#endif
775
776	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
777	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
778	{
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
780	&pVM->hm.s.vmx.HCPhysApicAccess);
781	if (RT_FAILURE(rc))
782	goto cleanup;
783	}
784
785	/*
786	* Initialize per-VCPU VT-x structures.
787	*/
788	for (VMCPUID i =0; i < pVM->cCpus; i++)
789	{
790	PVMCPU pVCpu = &pVM->aCpus[i];
791	AssertPtr(pVCpu);
792
793	/* Allocate the VM control structure (VMCS). */
794	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
795	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
796	if (RT_FAILURE(rc))
797	goto cleanup;
798
799	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
800	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
801	{
802	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
803	&pVCpu->hm.s.vmx.HCPhysVirtApic);
804	if (RT_FAILURE(rc))
805	goto cleanup;
806	}
807
808	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
809	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
810	{
811	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
812	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
813	if (RT_FAILURE(rc))
814	goto cleanup;
815	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
816	}
817
818	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
819	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
820	if (RT_FAILURE(rc))
821	goto cleanup;
822
823	/* Allocate the VM-exit MSR-load page for the host MSRs. */
824	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
825	if (RT_FAILURE(rc))
826	goto cleanup;
827	}
828
829	return VINF_SUCCESS;
830
831	cleanup:
832	hmR0VmxStructsFree(pVM);
833	return rc;
834	}
835
836
837	/**
838	* Does global VT-x initialization (called during module initialization).
839	*
840	* @returns VBox status code.
841	*/
842	VMMR0DECL(int) VMXR0GlobalInit(void)
843	{
844	/* Setup the main VM exit handlers. */
845	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
846	#ifdef DEBUG
847	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
848	Assert(s_apfnVMExitHandlers[i]);
849	#endif
850	return VINF_SUCCESS;
851	}
852
853
854	/**
855	* Does global VT-x termination (called during module termination).
856	*/
857	VMMR0DECL(void) VMXR0GlobalTerm()
858	{
859	/* Nothing to do currently. */
860	}
861
862
863	/**
864	* Sets up and activates VT-x on the current CPU.
865	*
866	* @returns VBox status code.
867	* @param pCpu Pointer to the global CPU info struct.
868	* @param pVM Pointer to the VM (can be NULL after a host resume
869	* operation).
870	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
871	* fEnabledByHost is true).
872	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
873	* @a fEnabledByHost is true).
874	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
875	* enable VT-x/AMD-V on the host.
876	*/
877	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
878	{
879	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
880	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
881
882	if (!fEnabledByHost)
883	{
884	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
885	if (RT_FAILURE(rc))
886	return rc;
887	}
888
889	/*
890	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
891	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
892	* each time while reusing a VPID after hitting the MaxASID limit once.
893	*/
894	if ( pVM
895	&& pVM->hm.s.vmx.fVpid
896	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
897	{
898	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
899	pCpu->fFlushAsidBeforeUse = false;
900	}
901	else
902	pCpu->fFlushAsidBeforeUse = true;
903
904	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
905	++pCpu->cTlbFlushes;
906
907	return VINF_SUCCESS;
908	}
909
910
911	/**
912	* Deactivates VT-x on the current CPU.
913	*
914	* @returns VBox status code.
915	* @param pCpu Pointer to the global CPU info struct.
916	* @param pvCpuPage Pointer to the VMXON region.
917	* @param HCPhysCpuPage Physical address of the VMXON region.
918	*/
919	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
920	{
921	NOREF(pCpu);
922	NOREF(pvCpuPage);
923	NOREF(HCPhysCpuPage);
924
925	hmR0VmxLeaveRootMode();
926	return VINF_SUCCESS;
927	}
928
929
930	/**
931	* Sets the permission bits for the specified MSR in the MSR bitmap.
932	*
933	* @param pVCpu Pointer to the VMCPU.
934	* @param ulMSR The MSR value.
935	* @param enmRead Whether reading this MSR causes a VM-exit.
936	* @param enmWrite Whether writing this MSR causes a VM-exit.
937	*/
938	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
939	{
940	unsigned ulBit;
941	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
942
943	/*
944	* Layout:
945	* 0x000 - 0x3ff - Low MSR read bits
946	* 0x400 - 0x7ff - High MSR read bits
947	* 0x800 - 0xbff - Low MSR write bits
948	* 0xc00 - 0xfff - High MSR write bits
949	*/
950	if (ulMsr <= 0x00001FFF)
951	{
952	/* Pentium-compatible MSRs */
953	ulBit = ulMsr;
954	}
955	else if ( ulMsr >= 0xC0000000
956	&& ulMsr <= 0xC0001FFF)
957	{
958	/* AMD Sixth Generation x86 Processor MSRs */
959	ulBit = (ulMsr - 0xC0000000);
960	pbMsrBitmap += 0x400;
961	}
962	else
963	{
964	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
965	return;
966	}
967
968	Assert(ulBit <= 0x1fff);
969	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
970	ASMBitSet(pbMsrBitmap, ulBit);
971	else
972	ASMBitClear(pbMsrBitmap, ulBit);
973
974	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
975	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
976	else
977	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
978	}
979
980
981	/**
982	* Flushes the TLB using EPT.
983	*
984	* @returns VBox status code.
985	* @param pVM Pointer to the VM.
986	* @param pVCpu Pointer to the VMCPU.
987	* @param enmFlush Type of flush.
988	*/
989	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
990	{
991	AssertPtr(pVM);
992	Assert(pVM->hm.s.fNestedPaging);
993
994	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
995
996	uint64_t descriptor[2];
997	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
998	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
999
1000	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1001	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1002	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1003	}
1004
1005
1006	/**
1007	* Flushes the TLB using VPID.
1008	*
1009	* @returns VBox status code.
1010	* @param pVM Pointer to the VM.
1011	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1012	* enmFlush).
1013	* @param enmFlush Type of flush.
1014	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1015	* on @a enmFlush).
1016	*/
1017	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1018	{
1019	AssertPtr(pVM);
1020	Assert(pVM->hm.s.vmx.fVpid);
1021
1022	uint64_t descriptor[2];
1023	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1024	{
1025	descriptor[0] = 0;
1026	descriptor[1] = 0;
1027	}
1028	else
1029	{
1030	AssertPtr(pVCpu);
1031	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1032	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1033	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1034	descriptor[1] = GCPtr;
1035	}
1036
1037	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1038	AssertMsg(rc == VINF_SUCCESS,
1039	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1040	if ( RT_SUCCESS(rc)
1041	&& pVCpu)
1042	{
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1044	}
1045	}
1046
1047
1048	/**
1049	* Invalidates a guest page by guest virtual address. Only relevant for
1050	* EPT/VPID, otherwise there is nothing really to invalidate.
1051	*
1052	* @returns VBox status code.
1053	* @param pVM Pointer to the VM.
1054	* @param pVCpu Pointer to the VMCPU.
1055	* @param GCVirt Guest virtual address of the page to invalidate.
1056	*/
1057	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1058	{
1059	AssertPtr(pVM);
1060	AssertPtr(pVCpu);
1061	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1062
1063	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1064	if (!fFlushPending)
1065	{
1066	/*
1067	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1068	* See @bugref{6043} and @bugref{6177}.
1069	*
1070	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1071	* function maybe called in a loop with individual addresses.
1072	*/
1073	if (pVM->hm.s.vmx.fVpid)
1074	{
1075	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1076	{
1077	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1078	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1079	}
1080	else
1081	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1082	}
1083	else if (pVM->hm.s.fNestedPaging)
1084	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1085	}
1086
1087	return VINF_SUCCESS;
1088	}
1089
1090
1091	/**
1092	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1093	* otherwise there is nothing really to invalidate.
1094	*
1095	* @returns VBox status code.
1096	* @param pVM Pointer to the VM.
1097	* @param pVCpu Pointer to the VMCPU.
1098	* @param GCPhys Guest physical address of the page to invalidate.
1099	*/
1100	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1101	{
1102	LogFlowFunc(("%RGp\n", GCPhys));
1103
1104	/*
1105	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1106	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1107	* This function might be called in a loop.
1108	*/
1109	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1110	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1117	* case where neither EPT nor VPID is supported by the CPU.
1118	*
1119	* @param pVM Pointer to the VM.
1120	* @param pVCpu Pointer to the VMCPU.
1121	*
1122	* @remarks Called with interrupts disabled.
1123	*/
1124	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1125	{
1126	NOREF(pVM);
1127	AssertPtr(pVCpu);
1128	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1130
1131	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1132	AssertPtr(pCpu);
1133
1134	pVCpu->hm.s.TlbShootdown.cPages = 0;
1135	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1136	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1137	pVCpu->hm.s.fForceTLBFlush = false;
1138	return;
1139	}
1140
1141
1142	/**
1143	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1144	*
1145	* @param pVM Pointer to the VM.
1146	* @param pVCpu Pointer to the VMCPU.
1147	* @remarks All references to "ASID" in this function pertains to "VPID" in
1148	* Intel's nomenclature. The reason is, to avoid confusion in compare
1149	* statements since the host-CPU copies are named "ASID".
1150	*
1151	* @remarks Called with interrupts disabled.
1152	*/
1153	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1154	{
1155	AssertPtr(pVM);
1156	AssertPtr(pVCpu);
1157	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1158	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1159	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1160
1161	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1162	AssertPtr(pCpu);
1163
1164	/*
1165	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1166	* This can happen both for start & resume due to long jumps back to ring-3.
1167	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1168	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1169	*/
1170	bool fNewASID = false;
1171	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1172	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1173	{
1174	pVCpu->hm.s.fForceTLBFlush = true;
1175	fNewASID = true;
1176	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1177	}
1178
1179	/*
1180	* Check for explicit TLB shootdowns.
1181	*/
1182	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1183	{
1184	pVCpu->hm.s.fForceTLBFlush = true;
1185	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1186	}
1187
1188	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1189	if (pVCpu->hm.s.fForceTLBFlush)
1190	{
1191	if (fNewASID)
1192	{
1193	++pCpu->uCurrentAsid;
1194	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1195	{
1196	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1197	pCpu->cTlbFlushes++;
1198	pCpu->fFlushAsidBeforeUse = true;
1199	}
1200
1201	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1202	if (pCpu->fFlushAsidBeforeUse)
1203	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1204	}
1205	else
1206	{
1207	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1208	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1209	else
1210	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1211	}
1212
1213	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1214	pVCpu->hm.s.fForceTLBFlush = false;
1215	}
1216	else
1217	{
1218	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1219	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1220	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1221	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1222
1223	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1224	* not be executed. See hmQueueInvlPage() where it is commented
1225	* out. Support individual entry flushing someday. */
1226	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1227	{
1228	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1229
1230	/*
1231	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1232	* as supported by the CPU.
1233	*/
1234	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1235	{
1236	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1237	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1238	}
1239	else
1240	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1241	}
1242	else
1243	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1244	}
1245	pVCpu->hm.s.TlbShootdown.cPages = 0;
1246	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1247
1248	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1249	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1250	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1251	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1252	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1253	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1254
1255	/* Update VMCS with the VPID. */
1256	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1257	AssertRC(rc);
1258	}
1259
1260
1261	/**
1262	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1263	*
1264	* @returns VBox status code.
1265	* @param pVM Pointer to the VM.
1266	* @param pVCpu Pointer to the VMCPU.
1267	*
1268	* @remarks Called with interrupts disabled.
1269	*/
1270	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1271	{
1272	AssertPtr(pVM);
1273	AssertPtr(pVCpu);
1274	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1275	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1276
1277	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1278	AssertPtr(pCpu);
1279
1280	/*
1281	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1282	* This can happen both for start & resume due to long jumps back to ring-3.
1283	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1284	*/
1285	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1286	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1287	{
1288	pVCpu->hm.s.fForceTLBFlush = true;
1289	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1290	}
1291
1292	/* Check for explicit TLB shootdown flushes. */
1293	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1294	{
1295	pVCpu->hm.s.fForceTLBFlush = true;
1296	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1297	}
1298
1299	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1300	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1301
1302	if (pVCpu->hm.s.fForceTLBFlush)
1303	{
1304	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1305	pVCpu->hm.s.fForceTLBFlush = false;
1306	}
1307	else
1308	{
1309	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1310	* not be executed. See hmQueueInvlPage() where it is commented
1311	* out. Support individual entry flushing someday. */
1312	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1313	{
1314	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1315	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1316	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1317	}
1318	else
1319	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1320	}
1321
1322	pVCpu->hm.s.TlbShootdown.cPages= 0;
1323	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1324	}
1325
1326
1327	/**
1328	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1329	*
1330	* @returns VBox status code.
1331	* @param pVM Pointer to the VM.
1332	* @param pVCpu Pointer to the VMCPU.
1333	*
1334	* @remarks Called with interrupts disabled.
1335	*/
1336	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1337	{
1338	AssertPtr(pVM);
1339	AssertPtr(pVCpu);
1340	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1341	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1342
1343	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1344
1345	/*
1346	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1347	* This can happen both for start & resume due to long jumps back to ring-3.
1348	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1349	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1350	*/
1351	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1352	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1353	{
1354	pVCpu->hm.s.fForceTLBFlush = true;
1355	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1356	}
1357
1358	/* Check for explicit TLB shootdown flushes. */
1359	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1360	{
1361	/*
1362	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1363	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1364	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1365	*/
1366	pVCpu->hm.s.fForceTLBFlush = true;
1367	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1368	}
1369
1370	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1371	if (pVCpu->hm.s.fForceTLBFlush)
1372	{
1373	++pCpu->uCurrentAsid;
1374	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1375	{
1376	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1377	pCpu->fFlushAsidBeforeUse = true;
1378	pCpu->cTlbFlushes++;
1379	}
1380
1381	pVCpu->hm.s.fForceTLBFlush = false;
1382	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1383	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1384	if (pCpu->fFlushAsidBeforeUse)
1385	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1386	}
1387	else
1388	{
1389	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1390	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1391	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1392	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1393
1394	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1395	* not be executed. See hmQueueInvlPage() where it is commented
1396	* out. Support individual entry flushing someday. */
1397	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1398	{
1399	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1400	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1401	{
1402	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1403	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1404	}
1405	else
1406	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1407	}
1408	else
1409	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1410	}
1411
1412	pVCpu->hm.s.TlbShootdown.cPages = 0;
1413	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1414
1415	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1416	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1417	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1418	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1419	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1420	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1421
1422	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1423	AssertRC(rc);
1424	}
1425
1426
1427	/**
1428	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1429	* TLB entries from the host TLB before VM-entry.
1430	*
1431	* @returns VBox status code.
1432	* @param pVM Pointer to the VM.
1433	*/
1434	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1435	{
1436	/*
1437	* Determine optimal flush type for nested paging.
1438	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1439	* guest execution (see hmR3InitFinalizeR0()).
1440	*/
1441	if (pVM->hm.s.fNestedPaging)
1442	{
1443	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1444	{
1445	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1446	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1447	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1448	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1449	else
1450	{
1451	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1452	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1454	}
1455
1456	/* Make sure the write-back cacheable memory type for EPT is supported. */
1457	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1458	{
1459	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1460	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1461	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1462	}
1463	}
1464	else
1465	{
1466	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1467	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1468	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1469	}
1470	}
1471
1472	/*
1473	* Determine optimal flush type for VPID.
1474	*/
1475	if (pVM->hm.s.vmx.fVpid)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1478	{
1479	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1480	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1481	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1482	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1483	else
1484	{
1485	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1486	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1487	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1489	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1490	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1491	pVM->hm.s.vmx.fVpid = false;
1492	}
1493	}
1494	else
1495	{
1496	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1497	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1499	pVM->hm.s.vmx.fVpid = false;
1500	}
1501	}
1502
1503	/*
1504	* Setup the handler for flushing tagged-TLBs.
1505	*/
1506	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1507	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1508	else if (pVM->hm.s.fNestedPaging)
1509	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1510	else if (pVM->hm.s.vmx.fVpid)
1511	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1512	else
1513	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	/**
1519	* Sets up pin-based VM-execution controls in the VMCS.
1520	*
1521	* @returns VBox status code.
1522	* @param pVM Pointer to the VM.
1523	* @param pVCpu Pointer to the VMCPU.
1524	*/
1525	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1526	{
1527	AssertPtr(pVM);
1528	AssertPtr(pVCpu);
1529
1530	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1531	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1532
1533	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1534	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1535	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1536
1537	/* Enable the VMX preemption timer. */
1538	if (pVM->hm.s.vmx.fUsePreemptTimer)
1539	{
1540	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1541	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1542	}
1543
1544	if ((val & zap) != val)
1545	{
1546	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1547	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1548	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1549	}
1550
1551	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1552	AssertRCReturn(rc, rc);
1553
1554	/* Update VCPU with the currently set pin-based VM-execution controls. */
1555	pVCpu->hm.s.vmx.u32PinCtls = val;
1556	return rc;
1557	}
1558
1559
1560	/**
1561	* Sets up processor-based VM-execution controls in the VMCS.
1562	*
1563	* @returns VBox status code.
1564	* @param pVM Pointer to the VM.
1565	* @param pVMCPU Pointer to the VMCPU.
1566	*/
1567	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1568	{
1569	AssertPtr(pVM);
1570	AssertPtr(pVCpu);
1571
1572	int rc = VERR_INTERNAL_ERROR_5;
1573	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1574	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1575
1576	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1577	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1578	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1579	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1580	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1581	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1582	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1583
1584	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1585	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1586	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1587	{
1588	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1589	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1590	}
1591
1592	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1593	if (!pVM->hm.s.fNestedPaging)
1594	{
1595	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1596	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1597	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1598	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1599	}
1600
1601	/* Use TPR shadowing if supported by the CPU. */
1602	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1603	{
1604	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1605	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1606	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1607	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1608	AssertRCReturn(rc, rc);
1609
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1611	/* CR8 writes causes a VM-exit based on TPR threshold. */
1612	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1613	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1614	}
1615	else
1616	{
1617	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1618	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1619	}
1620
1621	/* Use MSR-bitmaps if supported by the CPU. */
1622	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1623	{
1624	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1625
1626	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1627	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1628	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1629	AssertRCReturn(rc, rc);
1630
1631	/*
1632	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1633	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1634	*/
1635	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1636	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1637	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1638	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1639	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1640	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1641	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1642	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1643	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1644	}
1645
1646	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1647	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1648	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1649
1650	if ((val & zap) != val)
1651	{
1652	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1653	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1654	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1655	}
1656
1657	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1658	AssertRCReturn(rc, rc);
1659
1660	/* Update VCPU with the currently set processor-based VM-execution controls. */
1661	pVCpu->hm.s.vmx.u32ProcCtls = val;
1662
1663	/*
1664	* Secondary processor-based VM-execution controls.
1665	*/
1666	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1667	{
1668	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1669	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1670
1671	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1673
1674	if (pVM->hm.s.fNestedPaging)
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1676	else
1677	{
1678	/*
1679	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1680	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1681	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1682	*/
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1684	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1685	}
1686
1687	if (pVM->hm.s.vmx.fVpid)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1689
1690	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1691	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1692
1693	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1694	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1695	* done dynamically. */
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1697	{
1698	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1699	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1700	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1701	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1702	AssertRCReturn(rc, rc);
1703	}
1704
1705	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1706	{
1707	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	}
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1715	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1724	}
1725
1726	return VINF_SUCCESS;
1727	}
1728
1729
1730	/**
1731	* Sets up miscellaneous (everything other than Pin & Processor-based
1732	* VM-execution) control fields in the VMCS.
1733	*
1734	* @returns VBox status code.
1735	* @param pVM Pointer to the VM.
1736	* @param pVCpu Pointer to the VMCPU.
1737	*/
1738	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1739	{
1740	AssertPtr(pVM);
1741	AssertPtr(pVCpu);
1742
1743	int rc = VERR_GENERAL_FAILURE;
1744
1745	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1746	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1747
1748	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1749
1750	/*
1751	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1752	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1753	* We thus use the exception bitmap to control it rather than use both.
1754	*/
1755	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1756	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1757
1758	/** @todo Explore possibility of using IO-bitmaps. */
1759	/* All IO & IOIO instructions cause VM-exits. */
1760	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1762
1763	/* Setup MSR autoloading/autostoring. */
1764	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1765	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1766	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1767	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1768	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1770
1771	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1772	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1773	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1774	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1775
1776	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1777	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1778
1779	/* Setup debug controls */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1781	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1782	AssertRCReturn(rc, rc);
1783	return rc;
1784	}
1785
1786
1787	/**
1788	* Sets up the initial exception bitmap in the VMCS based on static conditions
1789	* (i.e. conditions that cannot ever change at runtime).
1790	*
1791	* @returns VBox status code.
1792	* @param pVM Pointer to the VM.
1793	* @param pVCpu Pointer to the VMCPU.
1794	*/
1795	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1796	{
1797	AssertPtr(pVM);
1798	AssertPtr(pVCpu);
1799
1800	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1801
1802	uint32_t u32XcptBitmap = 0;
1803
1804	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1805	if (!pVM->hm.s.fNestedPaging)
1806	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1807
1808	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1809	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1810	AssertRCReturn(rc, rc);
1811	return rc;
1812	}
1813
1814
1815	/**
1816	* Sets up the initial guest-state mask. The guest-state mask is consulted
1817	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1818	* for the nested virtualization case (as it would cause a VM-exit).
1819	*
1820	* @param pVCpu Pointer to the VMCPU.
1821	*/
1822	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1823	{
1824	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1825	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1826	return VINF_SUCCESS;
1827	}
1828
1829
1830	/**
1831	* Does per-VM VT-x initialization.
1832	*
1833	* @returns VBox status code.
1834	* @param pVM Pointer to the VM.
1835	*/
1836	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1837	{
1838	LogFlowFunc(("pVM=%p\n", pVM));
1839
1840	int rc = hmR0VmxStructsAlloc(pVM);
1841	if (RT_FAILURE(rc))
1842	{
1843	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1844	return rc;
1845	}
1846
1847	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1848	{
1849	PVMCPU pVCpu = &pVM->aCpus[i];
1850
1851	/* Current guest paging mode. */
1852	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1853	}
1854
1855	return VINF_SUCCESS;
1856	}
1857
1858
1859	/**
1860	* Does per-VM VT-x termination.
1861	*
1862	* @returns VBox status code.
1863	* @param pVM Pointer to the VM.
1864	*/
1865	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1866	{
1867	LogFlowFunc(("pVM=%p\n", pVM));
1868
1869	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1870	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1871	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1872	#endif
1873	hmR0VmxStructsFree(pVM);
1874	return VINF_SUCCESS;
1875	}
1876
1877
1878	/**
1879	* Sets up the VM for execution under VT-x.
1880	* This function is only called once per-VM during initalization.
1881	*
1882	* @returns VBox status code.
1883	* @param pVM Pointer to the VM.
1884	*/
1885	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1886	{
1887	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1888	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1889
1890	LogFlowFunc(("pVM=%p\n", pVM));
1891
1892	/*
1893	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1894	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1895	*/
1896	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1897	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1898	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1899	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1900	{
1901	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1902	return VERR_INTERNAL_ERROR;
1903	}
1904
1905	/* Initialize these always, see hmR3InitFinalizeR0().*/
1906	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1907	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1908
1909	/* Setup the tagged-TLB flush handlers. */
1910	int rc = hmR0VmxSetupTaggedTlb(pVM);
1911	if (RT_FAILURE(rc))
1912	{
1913	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1914	return rc;
1915	}
1916
1917	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1918	{
1919	PVMCPU pVCpu = &pVM->aCpus[i];
1920	AssertPtr(pVCpu);
1921	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1922
1923	/* Set revision dword at the beginning of the VMCS structure. */
1924	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1925
1926	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1927	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1928	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1929	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1930
1931	/* Load this VMCS as the current VMCS. */
1932	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1933	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1934	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1935
1936	/* Setup the pin-based VM-execution controls. */
1937	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Setup the processor-based VM-execution controls. */
1942	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the rest (miscellaneous) VM-execution controls. */
1947	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	/* Setup the initial exception bitmap. */
1952	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1953	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1954	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1955
1956	/* Setup the initial guest-state mask. */
1957	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1962	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1963	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1964	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1965	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1966	#endif
1967
1968	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1969	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1970	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1971	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1972
1973	/* Update the last error record for this VCPU. */
1974	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1975	}
1976
1977	return VINF_SUCCESS;
1978	}
1979
1980
1981	/**
1982	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1983	* the VMCS.
1984	*
1985	* @returns VBox status code.
1986	* @param pVM Pointer to the VM.
1987	* @param pVCpu Pointer to the VMCPU.
1988	*/
1989	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1990	{
1991	RTCCUINTREG uReg = ASMGetCR0();
1992	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1993
1994	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1995	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1996	if (VMX_IS_64BIT_HOST_MODE())
1997	{
1998	uint64_t uReg = hmR0Get64bitCR3();
1999	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2000	}
2001	else
2002	#endif
2003	{
2004	uReg = ASMGetCR3();
2005	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2006	}
2007
2008	uReg = ASMGetCR4();
2009	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2010	AssertRCReturn(rc, rc);
2011	return rc;
2012	}
2013
2014
2015	/**
2016	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2017	* the host-state area in the VMCS.
2018	*
2019	* @returns VBox status code.
2020	* @param pVM Pointer to the VM.
2021	* @param pVCpu Pointer to the VMCPU.
2022	*/
2023	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2024	{
2025	int rc = VERR_INTERNAL_ERROR_5;
2026	RTSEL uSelCS = 0;
2027	RTSEL uSelSS = 0;
2028	RTSEL uSelDS = 0;
2029	RTSEL uSelES = 0;
2030	RTSEL uSelFS = 0;
2031	RTSEL uSelGS = 0;
2032	RTSEL uSelTR = 0;
2033
2034	/*
2035	* Host Selector registers.
2036	*/
2037	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2038	if (VMX_IS_64BIT_HOST_MODE())
2039	{
2040	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2041	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2042	}
2043	else
2044	{
2045	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2046	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2047	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2048	}
2049	#else
2050	uSelCS = ASMGetCS();
2051	uSelSS = ASMGetSS();
2052	#endif
2053
2054	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2055	/** @todo Verify if we have any platform that actually run with DS or ES with
2056	* RPL != 0 in kernel space. */
2057	uSelDS = 0;
2058	uSelES = 0;
2059	uSelFS = 0;
2060	uSelGS = 0;
2061	uSelTR = ASMGetTR();
2062
2063	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2064	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2065	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2066	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2067	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2068	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2069	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2070	Assert(uSelCS != 0);
2071	Assert(uSelTR != 0);
2072
2073	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2074	#if 0
2075	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2076	Assert(uSelSS != 0);
2077	#endif
2078
2079	/* Write these host selector fields into the host-state area in the VMCS. */
2080	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2081	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2082	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2083	#if 0
2084	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2085	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2086	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2087	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2088	#endif
2089	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2090	AssertRCReturn(rc, rc);
2091
2092	/*
2093	* Host GDTR and IDTR.
2094	*/
2095	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2096	* be safe to -not- save and restore GDTR and IDTR in the assembly
2097	* code and just do it here and don't care if the limits are zapped on
2098	* VM-exit. */
2099	RTGDTR Gdtr;
2100	RT_ZERO(Gdtr);
2101	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2102	if (VMX_IS_64BIT_HOST_MODE())
2103	{
2104	X86XDTR64 Gtr64;
2105	X86XDTR64 Idtr64;
2106	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2107	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2108	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2109	Gdtr.cbGdt = Gdtr64.cb;
2110	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2111	}
2112	else
2113	#endif
2114	{
2115	RTIDTR Idtr;
2116	ASMGetGDTR(&Gdtr);
2117	ASMGetIDTR(&Idtr);
2118	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2119	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2120	}
2121	AssertRCReturn(rc, rc);
2122
2123	/*
2124	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2125	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2126	*/
2127	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2128	{
2129	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2130	return VERR_VMX_INVALID_HOST_STATE;
2131	}
2132
2133	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2134	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2135	if (VMX_IS_64BIT_HOST_MODE())
2136	{
2137	/* We need the 64-bit TR base for hybrid darwin. */
2138	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2139	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2140	}
2141	else
2142	#endif
2143	{
2144	uintptr_t uTRBase;
2145	#if HC_ARCH_BITS == 64
2146	uTRBase = X86DESC64_BASE(pDesc);
2147	#else
2148	uTRBase = X86DESC_BASE(pDesc);
2149	#endif
2150	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2151	}
2152	AssertRCReturn(rc, rc);
2153
2154	/*
2155	* Host FS base and GS base.
2156	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2157	* would take care of the bases. In 64-bit, the MSRs come into play.
2158	*/
2159	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2160	if (VMX_IS_64BIT_HOST_MODE())
2161	{
2162	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2163	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2164	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2165	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2166	AssertRCReturn(rc, rc);
2167	}
2168	#endif
2169	return rc;
2170	}
2171
2172
2173	/**
2174	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2175	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2176	* the host after every successful VM exit.
2177	*
2178	* @returns VBox status code.
2179	* @param pVM Pointer to the VM.
2180	* @param pVCpu Pointer to the VMCPU.
2181	*/
2182	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2183	{
2184	AssertPtr(pVCpu);
2185	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2186
2187	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2188	unsigned idxHostMsr = 0;
2189	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2190
2191	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2192	{
2193	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2194	pHostMsr->u32Reserved = 0;
2195	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2196	if (CPUMIsGuestInLongMode(pVCpu))
2197	{
2198	/* Must match the EFER value in our 64 bits switcher. */
2199	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2200	}
2201	else
2202	#endif
2203	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2204	pHostMsr++; idxHostMsr++;
2205	}
2206
2207	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2208	if (VMX_IS_64BIT_HOST_MODE())
2209	{
2210	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2211	pHostMsr->u32Reserved = 0;
2212	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2213	pHostMsr++; idxHostMsr++;
2214	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2215	pHostMsr->u32Reserved = 0;
2216	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2217	pHostMsr++; idxHostMsr++;
2218	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2219	pHostMsr->u32Reserved = 0;
2220	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2221	pHostMsr++; idxHostMsr++;
2222	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2223	#if 0
2224	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2225	pMsr->u32Reserved = 0;
2226	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2227	pHostMsr++; idxHostMsr++;
2228	#endif
2229	}
2230	#endif
2231
2232	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2233	if (RT_UNLIKELY(idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2234	{
2235	LogRel(("idxHostMsr=%u Cpu=%u\n", idxHostMsr, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2236	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237	}
2238
2239	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2240
2241	/*
2242	* Host Sysenter MSRs.
2243	*/
2244	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2245	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2246	if (VMX_IS_64BIT_HOST_MODE())
2247	{
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2249	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2250	}
2251	else
2252	{
2253	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2254	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2255	}
2256	#elif HC_ARCH_BITS == 32
2257	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2258	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2259	#else
2260	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2261	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2262	#endif
2263	AssertRCReturn(rc, rc);
2264
2265	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2266	* hmR0VmxSetupExitCtls() !! */
2267	return rc;
2268	}
2269
2270
2271	/**
2272	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2273	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2274	* controls".
2275	*
2276	* @returns VBox status code.
2277	* @param pVM Pointer to the VM.
2278	* @param pVCpu Pointer to the VMCPU.
2279	* @param pCtx Pointer to the guest-CPU context.
2280	*
2281	* @remarks No-long-jump zone!!!
2282	*/
2283	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2284	{
2285	int rc = VINF_SUCCESS;
2286	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2287	{
2288	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2289	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2290
2291	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2292	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2293
2294	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2295	if (CPUMIsGuestInLongModeEx(pCtx))
2296	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2297	else
2298	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2299
2300	/*
2301	* The following should not be set (since we're not in SMM mode):
2302	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2303	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2304	*/
2305
2306	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2307	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2308	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2309
2310	if ((val & zap) != val)
2311	{
2312	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2313	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2314	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2315	}
2316
2317	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2318	AssertRCReturn(rc, rc);
2319
2320	/* Update VCPU with the currently set VM-exit controls. */
2321	pVCpu->hm.s.vmx.u32EntryCtls = val;
2322	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2323	}
2324	return rc;
2325	}
2326
2327
2328	/**
2329	* Sets up the VM-exit controls in the VMCS.
2330	*
2331	* @returns VBox status code.
2332	* @param pVM Pointer to the VM.
2333	* @param pVCpu Pointer to the VMCPU.
2334	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2335	* out-of-sync. Make sure to update the required fields
2336	* before using them.
2337	*
2338	* @remarks requires EFER.
2339	*/
2340	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2341	{
2342	int rc = VINF_SUCCESS;
2343	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2344	{
2345	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2346	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2347
2348	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2349	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2350
2351	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2352	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2353	if (VMX_IS_64BIT_HOST_MODE())
2354	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2355	else
2356	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2357	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2358	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2359	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2360	else
2361	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2362	#endif
2363
2364	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2365	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2366
2367	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2368	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2369	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2370	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2371	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2372
2373	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2374	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2375
2376	if ((val & zap) != val)
2377	{
2378	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2379	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2380	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2381	}
2382
2383	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2384	AssertRCReturn(rc, rc);
2385
2386	/* Update VCPU with the currently set VM-exit controls. */
2387	pVCpu->hm.s.vmx.u32ExitCtls = val;
2388	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2389	}
2390	return rc;
2391	}
2392
2393
2394	/**
2395	* Loads the guest APIC and related state.
2396	*
2397	* @returns VBox status code.
2398	* @param pVM Pointer to the VM.
2399	* @param pVCpu Pointer to the VMCPU.
2400	* @param pCtx Pointer to the guest-CPU context.
2401	*/
2402	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2403	{
2404	int rc = VINF_SUCCESS;
2405	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2406	{
2407	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2408	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2409	{
2410	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2411
2412	bool fPendingIntr = false;
2413	uint8_t u8GuestTpr = 0;
2414	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2415	AssertRCReturn(rc, rc);
2416
2417	/*
2418	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2419	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2420	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2421	* the interrupt when we VM-exit for other reasons.
2422	*/
2423	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2424	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2425	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2426	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2427
2428	Log(("Getting TPR=%d Threshold=%u\n", pVCpu->hm.s.vmx.pbVirtApic[0x80], u32TprThreshold));
2429	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2430	AssertRCReturn(rc, rc);
2431
2432	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2433	if (pVM->hm.s.fTPRPatchingActive)
2434	{
2435	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2436	pCtx->msrLSTAR = u8GuestTpr;
2437	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2438	{
2439	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2440	if (fPendingIntr)
2441	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2442	else
2443	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444	}
2445	}
2446	}
2447
2448	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2449	}
2450	return rc;
2451	}
2452
2453
2454	/**
2455	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2456	* into the guest-state area in the VMCS.
2457	*
2458	* @param pVM Pointer to the VM.
2459	* @param pVCpu Pointer to the VMCPU.
2460	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2461	* out-of-sync. Make sure to update the required fields
2462	* before using them.
2463	*
2464	* @remarks No-long-jump zone!!!
2465	*/
2466	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2467	{
2468	/*
2469	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2470	* inhibit interrupts or clear any existing interrupt-inhibition.
2471	*/
2472	uint32_t uIntrState = 0;
2473	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2474	{
2475	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2476	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2477	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2478	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2479	{
2480	/*
2481	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2482	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2483	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2484	*/
2485	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2486	}
2487	else if (pMixedCtx->eflags.Bits.u1IF)
2488	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2489	else
2490	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2491	}
2492
2493	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2494	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2495	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2496	AssertRC(rc);
2497	}
2498
2499
2500	/**
2501	* Loads the guest's RIP into the guest-state area in the VMCS.
2502	*
2503	* @returns VBox status code.
2504	* @param pVM Pointer to the VM.
2505	* @param pVCpu Pointer to the VMCPU.
2506	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2507	* out-of-sync. Make sure to update the required fields
2508	* before using them.
2509	*
2510	* @remarks No-long-jump zone!!!
2511	*/
2512	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2513	{
2514	int rc = VINF_SUCCESS;
2515	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2516	{
2517	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2518	AssertRCReturn(rc, rc);
2519	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2520	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2521	}
2522	return rc;
2523	}
2524
2525
2526	/**
2527	* Loads the guest's RSP into the guest-state area in the VMCS.
2528	*
2529	* @returns VBox status code.
2530	* @param pVM Pointer to the VM.
2531	* @param pVCpu Pointer to the VMCPU.
2532	* @param pCtx Pointer to the guest-CPU context.
2533	*
2534	* @remarks No-long-jump zone!!!
2535	*/
2536	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2537	{
2538	int rc = VINF_SUCCESS;
2539	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2540	{
2541	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2542	AssertRCReturn(rc, rc);
2543	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2544	}
2545	return rc;
2546	}
2547
2548
2549	/**
2550	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2551	*
2552	* @returns VBox status code.
2553	* @param pVM Pointer to the VM.
2554	* @param pVCpu Pointer to the VMCPU.
2555	* @param pCtx Pointer to the guest-CPU context.
2556	*
2557	* @remarks No-long-jump zone!!!
2558	*/
2559	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2560	{
2561	int rc = VINF_SUCCESS;
2562	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2563	{
2564	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2565	Let us assert it as such and use native-width VMWRITE. */
2566	X86RFLAGS uRFlags = pCtx->rflags;
2567	Assert(uRFlags.u64 >> 32 == 0);
2568	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2569	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2570
2571	/*
2572	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2573	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2574	*/
2575	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2576	{
2577	Assert(pVM->hm.s.vmx.pRealModeTSS);
2578	Assert(PDMVmmDevHeapIsEnabled(pVM));
2579	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2580	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2581	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2582	}
2583
2584	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2585	AssertRCReturn(rc, rc);
2586
2587	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2588	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2589	}
2590	return rc;
2591	}
2592
2593
2594	/**
2595	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2596	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2597	* assembly code.
2598	*
2599	* @returns VBox status code.
2600	* @param pVM Pointer to the VM.
2601	* @param pVCpu Pointer to the VMCPU.
2602	* @param pCtx Pointer to the guest-CPU context.
2603	*
2604	* @remarks No-long-jump zone!!!
2605	*/
2606	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2607	{
2608	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2609	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2610	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2611	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2612	return rc;
2613	}
2614
2615
2616	/**
2617	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2618	* in the VMCS.
2619	*
2620	* @returns VBox status code.
2621	* @param pVM Pointer to the VM.
2622	* @param pVCpu Pointer to the VMCPU.
2623	* @param pCtx Pointer to the guest-CPU context.
2624	*
2625	* @remarks No-long-jump zone!!!
2626	*/
2627	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2628	{
2629	int rc = VINF_SUCCESS;
2630
2631	/*
2632	* Guest CR0.
2633	* Guest FPU.
2634	*/
2635	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2636	{
2637	uint64_t u64GuestCR0 = pCtx->cr0;
2638
2639	/* The guest's view (read access) of its CR0 is unblemished. */
2640	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2641	AssertRCReturn(rc, rc);
2642	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2643
2644	/* Setup VT-x's view of the guest CR0. */
2645	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2646	if (pVM->hm.s.fNestedPaging)
2647	{
2648	if (CPUMIsGuestPagingEnabledEx(pCtx))
2649	{
2650	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2651	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2653	}
2654	else
2655	{
2656	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2657	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2658	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2659	}
2660
2661	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2662	AssertRCReturn(rc, rc);
2663	}
2664	else
2665	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2666
2667	/*
2668	* Guest FPU bits.
2669	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2670	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2671	*/
2672	u64GuestCR0 \|= X86_CR0_NE;
2673	bool fInterceptNM = false;
2674	if (CPUMIsGuestFPUStateActive(pVCpu))
2675	{
2676	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2677	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2678	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2679	}
2680	else
2681	{
2682	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2683	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2684	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2685	}
2686
2687	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2688	bool fInterceptMF = false;
2689	if (!(pCtx->cr0 & X86_CR0_NE))
2690	fInterceptMF = true;
2691
2692	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2693	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2694	{
2695	Assert(PDMVmmDevHeapIsEnabled(pVM));
2696	Assert(pVM->hm.s.vmx.pRealModeTSS);
2697	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2698	fInterceptNM = true;
2699	fInterceptMF = true;
2700	}
2701	else
2702	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2703
2704	if (fInterceptNM)
2705	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2706	else
2707	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2708
2709	if (fInterceptMF)
2710	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2711	else
2712	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2713
2714	/* Additional intercepts for debugging, define these yourself explicitly. */
2715	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2716	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2717	\| RT_BIT(X86_XCPT_DB)
2718	\| RT_BIT(X86_XCPT_DE)
2719	\| RT_BIT(X86_XCPT_NM)
2720	\| RT_BIT(X86_XCPT_UD)
2721	\| RT_BIT(X86_XCPT_NP)
2722	\| RT_BIT(X86_XCPT_SS)
2723	\| RT_BIT(X86_XCPT_GP)
2724	\| RT_BIT(X86_XCPT_PF)
2725	\| RT_BIT(X86_XCPT_MF);
2726	#elif defined(VBOX_ALWAYS_TRAP_PF)
2727	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2728	#endif
2729
2730	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2731
2732	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2733	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2734	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2735	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2736	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2737	else
2738	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2739
2740	u64GuestCR0 \|= uSetCR0;
2741	u64GuestCR0 &= uZapCR0;
2742	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2743
2744	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2745	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2746	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2747	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2748
2749	/*
2750	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2751	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2752	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2753	*/
2754	uint64_t u64CR0Mask = 0;
2755	u64CR0Mask = X86_CR0_PE
2756	\| X86_CR0_WP
2757	\| X86_CR0_PG
2758	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2759	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2760	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2761
2762	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2763	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2764	u64CR0Mask &= ~X86_CR0_PE;
2765
2766	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2767	if (fInterceptNM)
2768	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2769	else
2770	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2771
2772	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2773	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2774	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2775	AssertRCReturn(rc, rc);
2776
2777	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2778	}
2779
2780	/*
2781	* Guest CR2.
2782	* It's always loaded in the assembler code. Nothing to do here.
2783	*/
2784
2785	/*
2786	* Guest CR3.
2787	*/
2788	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2789	{
2790	uint64_t u64GuestCR3 = 0;
2791	if (pVM->hm.s.fNestedPaging)
2792	{
2793	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2794
2795	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2796	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2797	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2798	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2799
2800	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2801	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2802	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2803
2804	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2805	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2806	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2807	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2808
2809	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2810	AssertRCReturn(rc, rc);
2811	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2812
2813	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2814	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2815	{
2816	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2817	if (CPUMIsGuestInPAEModeEx(pCtx))
2818	{
2819	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2820	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2821	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2822	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2823	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2824	AssertRCReturn(rc, rc);
2825	}
2826
2827	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2828	have Unrestricted Execution to handle the guest when it's not using paging. */
2829	u64GuestCR3 = pCtx->cr3;
2830	}
2831	else
2832	{
2833	/*
2834	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2835	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2836	* EPT takes care of translating it to host-physical addresses.
2837	*/
2838	RTGCPHYS GCPhys;
2839	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2840	Assert(PDMVmmDevHeapIsEnabled(pVM));
2841
2842	/* We obtain it here every time as the guest could have relocated this PCI region. */
2843	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2844	AssertRCReturn(rc, rc);
2845
2846	u64GuestCR3 = GCPhys;
2847	}
2848	}
2849	else
2850	{
2851	/* Non-nested paging case, just use the hypervisor's CR3. */
2852	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2853	}
2854
2855	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2856	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2857	AssertRCReturn(rc, rc);
2858
2859	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2860	}
2861
2862	/*
2863	* Guest CR4.
2864	*/
2865	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2866	{
2867	uint64_t u64GuestCR4 = pCtx->cr4;
2868
2869	/* The guest's view of its CR4 is unblemished. */
2870	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2871	AssertRCReturn(rc, rc);
2872	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2873
2874	/* Setup VT-x's view of the guest CR4. */
2875	/*
2876	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2877	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2878	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2879	*/
2880	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2881	{
2882	Assert(pVM->hm.s.vmx.pRealModeTSS);
2883	Assert(PDMVmmDevHeapIsEnabled(pVM));
2884	u64GuestCR4 &= ~X86_CR4_VME;
2885	}
2886
2887	if (pVM->hm.s.fNestedPaging)
2888	{
2889	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2890	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2891	{
2892	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2893	u64GuestCR4 \|= X86_CR4_PSE;
2894	/* Our identity mapping is a 32 bits page directory. */
2895	u64GuestCR4 &= ~X86_CR4_PAE;
2896	}
2897	/* else use guest CR4.*/
2898	}
2899	else
2900	{
2901	/*
2902	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2903	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2904	*/
2905	switch (pVCpu->hm.s.enmShadowMode)
2906	{
2907	case PGMMODE_REAL: /* Real-mode. */
2908	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2909	case PGMMODE_32_BIT: /* 32-bit paging. */
2910	{
2911	u64GuestCR4 &= ~X86_CR4_PAE;
2912	break;
2913	}
2914
2915	case PGMMODE_PAE: /* PAE paging. */
2916	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2917	{
2918	u64GuestCR4 \|= X86_CR4_PAE;
2919	break;
2920	}
2921
2922	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2923	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2924	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2925	break;
2926	#endif
2927	default:
2928	AssertFailed();
2929	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2930	}
2931	}
2932
2933	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2934	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2935	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2936	u64GuestCR4 \|= uSetCR4;
2937	u64GuestCR4 &= uZapCR4;
2938
2939	/* Write VT-x's view of the guest CR4 into the VMCS. */
2940	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2941	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2942
2943	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2944	uint64_t u64CR4Mask = 0;
2945	u64CR4Mask = X86_CR4_VME
2946	\| X86_CR4_PAE
2947	\| X86_CR4_PGE
2948	\| X86_CR4_PSE
2949	\| X86_CR4_VMXE;
2950	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2951	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2952	AssertRCReturn(rc, rc);
2953
2954	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2955	}
2956	return rc;
2957	}
2958
2959
2960	/**
2961	* Loads the guest debug registers into the guest-state area in the VMCS.
2962	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2963	*
2964	* @returns VBox status code.
2965	* @param pVM Pointer to the VM.
2966	* @param pVCpu Pointer to the VMCPU.
2967	* @param pCtx Pointer to the guest-CPU context.
2968	*
2969	* @remarks No-long-jump zone!!!
2970	*/
2971	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2972	{
2973	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2974	return VINF_SUCCESS;
2975
2976	#ifdef DEBUG
2977	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2978	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2979	{
2980	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2981	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2982	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2983	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2984	}
2985	#endif
2986
2987	int rc = VERR_INTERNAL_ERROR_5;
2988	bool fInterceptDB = false;
2989	bool fInterceptMovDRx = false;
2990	if (DBGFIsStepping(pVCpu))
2991	{
2992	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2993	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2994	{
2995	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2996	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2997	AssertRCReturn(rc, rc);
2998	Assert(fInterceptDB == false);
2999	}
3000	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3001	{
3002	Assert(fInterceptDB == false);
3003	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3004	* just continuing? */
3005	}
3006	else
3007	fInterceptDB = true;
3008	}
3009	else
3010	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3011
3012	/*
3013	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3014	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3015	* The same for the hypervisor DRx registers, priority is for the guest here.
3016	*/
3017	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3018	&& !CPUMIsGuestDebugStateActive(pVCpu))
3019	{
3020	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3021	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3022	AssertRC(rc);
3023	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3024	Assert(fInterceptMovDRx == false);
3025	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3026	}
3027	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3028	&& !CPUMIsHyperDebugStateActive(pVCpu))
3029	{
3030	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3031	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3032	AssertRC(rc);
3033	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3034	fInterceptMovDRx = true;
3035	}
3036	else
3037	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3038
3039	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3040	if (fInterceptDB)
3041	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3042	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3043	{
3044	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3045	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3046	#endif
3047	}
3048
3049	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3050	if (fInterceptMovDRx)
3051	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3052	else
3053	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3054
3055	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3056	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3057
3058	/* The guest's view of its DR7 is unblemished. */
3059	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3060
3061	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3062	return rc;
3063	}
3064
3065
3066	#ifdef DEBUG
3067	/**
3068	* Debug function to validate segment registers.
3069	*/
3070	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3071	{
3072	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3073	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3074	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3075	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3076	{
3077	/* Protected mode checks */
3078	/* CS */
3079	Assert(pCtx->cs.Attr.n.u1Present);
3080	Assert(!(pCtx->cs.Attr.u & 0xf00));
3081	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3082	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3083	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3084	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3085	\|\| (pCtx->cs.Attr.n.u1Granularity));
3086	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3087	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3088	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3089	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3090	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3091	else
3092	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3093	/* SS */
3094	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3095	{
3096	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3097	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3098	Assert(pCtx->ss.Attr.n.u1Present);
3099	Assert(!(pCtx->ss.Attr.u & 0xf00));
3100	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3101	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3102	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3103	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3104	\|\| (pCtx->ss.Attr.n.u1Granularity));
3105	}
3106	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3107	/* CR0 might not be up-to-date here always, hence disabled. */
3108	#if 0
3109	if (!pCtx->cr0 & X86_CR0_PE)
3110	Assert(!pCtx->ss.Attr.n.u2Dpl);
3111	#endif
3112	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3113	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3114	{
3115	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3116	Assert(pCtx->ds.Attr.n.u1Present);
3117	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3118	Assert(!(pCtx->ds.Attr.u & 0xf00));
3119	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3120	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3121	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3122	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3123	\|\| (pCtx->ds.Attr.n.u1Granularity));
3124	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3125	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3126	}
3127	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3128	{
3129	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3130	Assert(pCtx->es.Attr.n.u1Present);
3131	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3132	Assert(!(pCtx->es.Attr.u & 0xf00));
3133	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3134	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3135	\|\| !(pCtx->es.Attr.n.u1Granularity));
3136	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3137	\|\| (pCtx->es.Attr.n.u1Granularity));
3138	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3139	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3140	}
3141	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3142	{
3143	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3144	Assert(pCtx->fs.Attr.n.u1Present);
3145	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3146	Assert(!(pCtx->fs.Attr.u & 0xf00));
3147	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3148	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3149	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3150	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3151	\|\| (pCtx->fs.Attr.n.u1Granularity));
3152	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3153	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3154	}
3155	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3156	{
3157	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3158	Assert(pCtx->gs.Attr.n.u1Present);
3159	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3160	Assert(!(pCtx->gs.Attr.u & 0xf00));
3161	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3162	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3163	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3164	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3165	\|\| (pCtx->gs.Attr.n.u1Granularity));
3166	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3167	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3168	}
3169	/* 64-bit capable CPUs. */
3170	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3171	Assert(!(pCtx->cs.u64Base >> 32));
3172	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3173	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3174	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3175	# endif
3176	}
3177	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3178	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3179	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3180	{
3181	/* Real and v86 mode checks. */
3182	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3183	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3184	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3185	{
3186	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3187	}
3188	else
3189	{
3190	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3191	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3192	}
3193
3194	/* CS */
3195	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3196	Assert(pCtx->cs.u32Limit == 0xffff);
3197	Assert(u32CSAttr == 0xf3);
3198	/* SS */
3199	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3200	Assert(pCtx->ss.u32Limit == 0xffff);
3201	Assert(u32SSAttr == 0xf3);
3202	/* DS */
3203	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3204	Assert(pCtx->ds.u32Limit == 0xffff);
3205	Assert(u32DSAttr == 0xf3);
3206	/* ES */
3207	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3208	Assert(pCtx->es.u32Limit == 0xffff);
3209	Assert(u32ESAttr == 0xf3);
3210	/* FS */
3211	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3212	Assert(pCtx->fs.u32Limit == 0xffff);
3213	Assert(u32FSAttr == 0xf3);
3214	/* GS */
3215	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3216	Assert(pCtx->gs.u32Limit == 0xffff);
3217	Assert(u32GSAttr == 0xf3);
3218	/* 64-bit capable CPUs. */
3219	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3220	Assert(!(pCtx->cs.u64Base >> 32));
3221	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3222	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3223	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3224	# endif
3225	}
3226	}
3227	#endif /* DEBUG */
3228
3229
3230	/**
3231	* Writes a guest segment register into the guest-state area in the VMCS.
3232	*
3233	* @returns VBox status code.
3234	* @param pVM Pointer to the VM.
3235	* @param pVCpu Pointer to the VMCPU.
3236	* @param idxSel Index of the selector in the VMCS.
3237	* @param idxLimit Index of the segment limit in the VMCS.
3238	* @param idxBase Index of the segment base in the VMCS.
3239	* @param idxAccess Index of the access rights of the segment in the VMCS.
3240	* @param pSelReg Pointer to the segment selector.
3241	* @param pCtx Pointer to the guest-CPU context.
3242	*
3243	* @remarks No-long-jump zone!!!
3244	*/
3245	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3246	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3247	{
3248	int rc;
3249	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3250	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3251	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3252	AssertRCReturn(rc, rc);
3253
3254	uint32_t u32Access = pSelReg->Attr.u;
3255	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3256	{
3257	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3258	u32Access = 0xf3;
3259	Assert(pVM->hm.s.vmx.pRealModeTSS);
3260	Assert(PDMVmmDevHeapIsEnabled(pVM));
3261	}
3262	else
3263	{
3264	/*
3265	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3266	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3267	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3268	* loaded in protected-mode have their attribute as 0.
3269	*/
3270	if (!u32Access)
3271	u32Access = VMX_SEL_UNUSABLE;
3272	}
3273
3274	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3275	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3276	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3277
3278	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3279	AssertRCReturn(rc, rc);
3280	return rc;
3281	}
3282
3283
3284	/**
3285	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3286	* into the guest-state area in the VMCS.
3287	*
3288	* @returns VBox status code.
3289	* @param pVM Pointer to the VM.
3290	* @param pVCPU Pointer to the VMCPU.
3291	* @param pCtx Pointer to the guest-CPU context.
3292	*
3293	* @remarks No-long-jump zone!!!
3294	* @remarks Requires RFLAGS (for debug assertions).
3295	*/
3296	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3297	{
3298	int rc = VERR_INTERNAL_ERROR_5;
3299
3300	/*
3301	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3302	*/
3303	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3304	{
3305	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3306	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3307	{
3308	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3309	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3310	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3311	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3312	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3313	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3314	}
3315
3316	#ifdef VBOX_WITH_REM
3317	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3318	{
3319	Assert(pVM->hm.s.vmx.pRealModeTSS);
3320	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3321	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3322	{
3323	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3324	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3325	&& enmGuestMode >= PGMMODE_PROTECTED)
3326	{
3327	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3328	in real-mode (e.g. OpenBSD 4.0) */
3329	REMFlushTBs(pVM);
3330	Log2(("Switch to protected mode detected!\n"));
3331	}
3332	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3333	}
3334	}
3335	#endif
3336	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3337	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3338	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3339	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3340	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3341	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3342	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3343	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3344	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3345	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3346	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3347	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3348	AssertRCReturn(rc, rc);
3349
3350	#ifdef DEBUG
3351	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3352	#endif
3353	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3354	}
3355
3356	/*
3357	* Guest TR.
3358	*/
3359	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3360	{
3361	/*
3362	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3363	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3364	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3365	*/
3366	uint16_t u16Sel = 0;
3367	uint32_t u32Limit = 0;
3368	uint64_t u64Base = 0;
3369	uint32_t u32AccessRights = 0;
3370
3371	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3372	{
3373	u16Sel = pCtx->tr.Sel;
3374	u32Limit = pCtx->tr.u32Limit;
3375	u64Base = pCtx->tr.u64Base;
3376	u32AccessRights = pCtx->tr.Attr.u;
3377	}
3378	else
3379	{
3380	Assert(pVM->hm.s.vmx.pRealModeTSS);
3381	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3382
3383	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3384	RTGCPHYS GCPhys;
3385	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3386	AssertRCReturn(rc, rc);
3387
3388	X86DESCATTR DescAttr;
3389	DescAttr.u = 0;
3390	DescAttr.n.u1Present = 1;
3391	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3392
3393	u16Sel = 0;
3394	u32Limit = HM_VTX_TSS_SIZE;
3395	u64Base = GCPhys; /* in real-mode phys = virt. */
3396	u32AccessRights = DescAttr.u;
3397	}
3398
3399	/* Validate. */
3400	Assert(!(u16Sel & RT_BIT(2)));
3401	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3402	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3403	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3404	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3405	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3406	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3407	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3408	Assert( (u32Limit & 0xfff) == 0xfff
3409	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3410	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3411	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3412
3413	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3414	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3415	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3416	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3417	AssertRCReturn(rc, rc);
3418
3419	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3421	}
3422
3423	/*
3424	* Guest GDTR.
3425	*/
3426	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3427	{
3428	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3429	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3430	AssertRCReturn(rc, rc);
3431
3432	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3433	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3434	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3435	}
3436
3437	/*
3438	* Guest LDTR.
3439	*/
3440	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3441	{
3442	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3443	uint32_t u32Access = 0;
3444	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3445	u32Access = VMX_SEL_UNUSABLE;
3446	else
3447	u32Access = pCtx->ldtr.Attr.u;
3448
3449	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3450	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3451	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3452	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3453	AssertRCReturn(rc, rc);
3454
3455	/* Validate. */
3456	if (!(u32Access & VMX_SEL_UNUSABLE))
3457	{
3458	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3459	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3460	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3461	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3462	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3463	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3464	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3465	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3466	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3467	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3468	}
3469
3470	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3471	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3472	}
3473
3474	/*
3475	* Guest IDTR.
3476	*/
3477	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3478	{
3479	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3480	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3481	AssertRCReturn(rc, rc);
3482
3483	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3484	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3485	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3486	}
3487
3488	/*
3489	* Guest FS & GS base MSRs.
3490	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3491	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3492	* are only available in 64-bit mode.
3493	*/
3494	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3495	* should not be necessary to do it in assembly again. */
3496	if (CPUMIsGuestInLongModeEx(pCtx))
3497	{
3498	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3499	{
3500	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3501	AssertRCReturn(rc, rc);
3502	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3503	}
3504
3505	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3506	{
3507	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3508	AssertRCReturn(rc, rc);
3509	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3510	}
3511	}
3512	else
3513	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3514
3515	return VINF_SUCCESS;
3516	}
3517
3518
3519	/**
3520	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3521	* areas. These MSRs will automatically be loaded to the host CPU on every
3522	* successful VM entry and stored from the host CPU on every successful VM exit.
3523	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3524	*
3525	* @returns VBox status code.
3526	* @param pVM Pointer to the VM.
3527	* @param pVCpu Pointer to the VMCPU.
3528	* @param pCtx Pointer to the guest-CPU context.
3529	*
3530	* @remarks No-long-jump zone!!!
3531	*/
3532	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3533	{
3534	AssertPtr(pVCpu);
3535	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3536
3537	/*
3538	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3539	*/
3540	int rc = VINF_SUCCESS;
3541	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3542	{
3543	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3544	unsigned cGuestMsrs = 0;
3545
3546	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3547	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3548	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3549	if (fSupportsNX \|\| fSupportsLongMode)
3550	{
3551	/** @todo support save IA32_EFER, i.e.
3552	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3553	* guest EFER need not be part of the VM-entry MSR-load area. */
3554	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3555	pGuestMsr->u32Reserved = 0;
3556	pGuestMsr->u64Value = pCtx->msrEFER;
3557	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3558	if (!CPUMIsGuestInLongModeEx(pCtx))
3559	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3560	pGuestMsr++; cGuestMsrs++;
3561	if (fSupportsLongMode)
3562	{
3563	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3564	pGuestMsr->u32Reserved = 0;
3565	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3566	pGuestMsr++; cGuestMsrs++;
3567	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3568	pGuestMsr->u32Reserved = 0;
3569	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3570	pGuestMsr++; cGuestMsrs++;
3571	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3572	pGuestMsr->u32Reserved = 0;
3573	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3574	pGuestMsr++; cGuestMsrs++;
3575	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3576	#if 0
3577	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3578	pGuestMsr->u32Reserved = 0;
3579	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3580	pGuestMsr++; cGuestMsrs++;
3581	#endif
3582	}
3583	}
3584
3585	/*
3586	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3587	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3588	*/
3589	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3590	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3591	{
3592	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3593	pGuestMsr->u32Reserved = 0;
3594	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3595	AssertRCReturn(rc, rc);
3596	pGuestMsr++; cGuestMsrs++;
3597	}
3598
3599	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3600	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3601	{
3602	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3603	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3604	}
3605
3606	/* Update the VCPU's copy of the guest MSR count. */
3607	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3608	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3609	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3610	AssertRCReturn(rc, rc);
3611
3612	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3613	}
3614
3615	/*
3616	* Guest Sysenter MSRs.
3617	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3618	* VM exits on WRMSRs for these MSRs.
3619	*/
3620	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3621	{
3622	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3623	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3624	}
3625	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3626	{
3627	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3628	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3629	}
3630	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3631	{
3632	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3633	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3634	}
3635	AssertRCReturn(rc, rc);
3636
3637	return rc;
3638	}
3639
3640
3641	/**
3642	* Loads the guest activity state into the guest-state area in the VMCS.
3643	*
3644	* @returns VBox status code.
3645	* @param pVM Pointer to the VM.
3646	* @param pVCpu Pointer to the VMCPU.
3647	* @param pCtx Pointer to the guest-CPU context.
3648	*
3649	* @remarks No-long-jump zone!!!
3650	*/
3651	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3652	{
3653	/** @todo See if we can make use of other states, e.g.
3654	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3655	int rc = VINF_SUCCESS;
3656	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3657	{
3658	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3659	AssertRCReturn(rc, rc);
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3661	}
3662	return rc;
3663	}
3664
3665
3666	/**
3667	* Sets up the appropriate function to run guest code.
3668	*
3669	* @returns VBox status code.
3670	* @param pVM Pointer to the VM.
3671	* @param pVCpu Pointer to the VMCPU.
3672	* @param pCtx Pointer to the guest-CPU context.
3673	*
3674	* @remarks No-long-jump zone!!!
3675	*/
3676	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3677	{
3678	if (CPUMIsGuestInLongModeEx(pCtx))
3679	{
3680	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3681	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3682	#endif
3683	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3684	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3685	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3686	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3687	#else
3688	/* 64-bit host or hybrid host. */
3689	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3690	#endif
3691	}
3692	else
3693	{
3694	/* Guest is not in long mode, use the 32-bit handler. */
3695	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3696	}
3697	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3698	return VINF_SUCCESS;
3699	}
3700
3701
3702	/**
3703	* Wrapper for running the guest code in VT-x.
3704	*
3705	* @returns VBox strict status code.
3706	* @param pVM Pointer to the VM.
3707	* @param pVCpu Pointer to the VMCPU.
3708	* @param pCtx Pointer to the guest-CPU context.
3709	*
3710	* @remarks No-long-jump zone!!!
3711	*/
3712	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3713	{
3714	/*
3715	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3716	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3717	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3718	*/
3719	#ifdef VBOX_WITH_KERNEL_USING_XMM
3720	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3721	#else
3722	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3723	#endif
3724	}
3725
3726
3727	/**
3728	* Report world-switch error and dump some useful debug info.
3729	*
3730	* @param pVM Pointer to the VM.
3731	* @param pVCpu Pointer to the VMCPU.
3732	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3733	* @param pCtx Pointer to the guest-CPU context.
3734	* @param pVmxTransient Pointer to the VMX transient structure (only
3735	* exitReason updated).
3736	*/
3737	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3738	{
3739	Assert(pVM);
3740	Assert(pVCpu);
3741	Assert(pCtx);
3742	Assert(pVmxTransient);
3743	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3744
3745	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3746	switch (rcVMRun)
3747	{
3748	case VERR_VMX_INVALID_VMXON_PTR:
3749	AssertFailed();
3750	break;
3751	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3752	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3753	{
3754	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3755	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3756	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3757	AssertRC(rc);
3758
3759	#ifdef VBOX_STRICT
3760	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3761	pVmxTransient->uExitReason));
3762	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3763	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3764	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3765	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3766	else
3767	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3768
3769	/* VMX control bits. */
3770	uint32_t u32Val;
3771	uint64_t u64Val;
3772	RTHCUINTREG uHCReg;
3773	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3774	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3775	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3776	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3777	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3778	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3779	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3780	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3781	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3782	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3783	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3784	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3785	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3786	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3787	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3788	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3789	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3790	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3791	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3792	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3793	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3794	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3796	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3798	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3800	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3802	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3804	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3805	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3806	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3807	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3808	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3809	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3810	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3811	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3812	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3813	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3814	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3815
3816	/* Guest bits. */
3817	RTGCUINTREG uGCReg;
3818	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3819	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3820	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3821	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3822	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3823	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3824	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3826
3827	/* Host bits. */
3828	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3829	Log(("Host CR0 %#RHr\n", uHCReg));
3830	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3831	Log(("Host CR3 %#RHr\n", uHCReg));
3832	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3833	Log(("Host CR4 %#RHr\n", uHCReg));
3834
3835	RTGDTR HostGdtr;
3836	PCX86DESCHC pDesc;
3837	ASMGetGDTR(&HostGdtr);
3838	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3839	Log(("Host CS %#08x\n", u32Val));
3840	if (u32Val < HostGdtr.cbGdt)
3841	{
3842	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3843	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3844	}
3845
3846	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3847	Log(("Host DS %#08x\n", u32Val));
3848	if (u32Val < HostGdtr.cbGdt)
3849	{
3850	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3851	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3852	}
3853
3854	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3855	Log(("Host ES %#08x\n", u32Val));
3856	if (u32Val < HostGdtr.cbGdt)
3857	{
3858	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3859	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3860	}
3861
3862	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3863	Log(("Host FS %#08x\n", u32Val));
3864	if (u32Val < HostGdtr.cbGdt)
3865	{
3866	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3867	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3868	}
3869
3870	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3871	Log(("Host GS %#08x\n", u32Val));
3872	if (u32Val < HostGdtr.cbGdt)
3873	{
3874	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3875	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3876	}
3877
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3879	Log(("Host SS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3887	Log(("Host TR %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3892	}
3893
3894	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3895	Log(("Host TR Base %#RHv\n", uHCReg));
3896	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3897	Log(("Host GDTR Base %#RHv\n", uHCReg));
3898	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3899	Log(("Host IDTR Base %#RHv\n", uHCReg));
3900	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3901	Log(("Host SYSENTER CS %#08x\n", u32Val));
3902	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3903	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3904	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3905	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3906	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3907	Log(("Host RSP %#RHv\n", uHCReg));
3908	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3909	Log(("Host RIP %#RHv\n", uHCReg));
3910	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3911	if (VMX_IS_64BIT_HOST_MODE())
3912	{
3913	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3914	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3915	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3916	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3917	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3918	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3919	}
3920	# endif
3921	#endif /* VBOX_STRICT */
3922	break;
3923	}
3924
3925	default:
3926	/* Impossible */
3927	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3928	break;
3929	}
3930	NOREF(pVM);
3931	}
3932
3933
3934	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3935	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3936	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3937	#endif
3938
3939	#ifdef VBOX_STRICT
3940	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3941	{
3942	switch (idxField)
3943	{
3944	case VMX_VMCS_GUEST_RIP:
3945	case VMX_VMCS_GUEST_RSP:
3946	case VMX_VMCS_GUEST_RFLAGS:
3947	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3948	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3949	case VMX_VMCS_GUEST_CR0:
3950	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3951	case VMX_VMCS_GUEST_CR4:
3952	case VMX_VMCS_GUEST_DR7:
3953	case VMX_VMCS32_GUEST_SYSENTER_CS:
3954	case VMX_VMCS_GUEST_SYSENTER_EIP:
3955	case VMX_VMCS_GUEST_SYSENTER_ESP:
3956	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3957	case VMX_VMCS_GUEST_GDTR_BASE:
3958	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3959	case VMX_VMCS_GUEST_IDTR_BASE:
3960	case VMX_VMCS16_GUEST_FIELD_CS:
3961	case VMX_VMCS32_GUEST_CS_LIMIT:
3962	case VMX_VMCS_GUEST_CS_BASE:
3963	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3964	case VMX_VMCS16_GUEST_FIELD_DS:
3965	case VMX_VMCS32_GUEST_DS_LIMIT:
3966	case VMX_VMCS_GUEST_DS_BASE:
3967	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
3968	case VMX_VMCS16_GUEST_FIELD_ES:
3969	case VMX_VMCS32_GUEST_ES_LIMIT:
3970	case VMX_VMCS_GUEST_ES_BASE:
3971	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
3972	case VMX_VMCS16_GUEST_FIELD_FS:
3973	case VMX_VMCS32_GUEST_FS_LIMIT:
3974	case VMX_VMCS_GUEST_FS_BASE:
3975	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
3976	case VMX_VMCS16_GUEST_FIELD_GS:
3977	case VMX_VMCS32_GUEST_GS_LIMIT:
3978	case VMX_VMCS_GUEST_GS_BASE:
3979	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
3980	case VMX_VMCS16_GUEST_FIELD_SS:
3981	case VMX_VMCS32_GUEST_SS_LIMIT:
3982	case VMX_VMCS_GUEST_SS_BASE:
3983	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
3984	case VMX_VMCS16_GUEST_FIELD_LDTR:
3985	case VMX_VMCS32_GUEST_LDTR_LIMIT:
3986	case VMX_VMCS_GUEST_LDTR_BASE:
3987	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
3988	case VMX_VMCS16_GUEST_FIELD_TR:
3989	case VMX_VMCS32_GUEST_TR_LIMIT:
3990	case VMX_VMCS_GUEST_TR_BASE:
3991	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
3992	case VMX_VMCS32_RO_EXIT_REASON:
3993	case VMX_VMCS32_RO_VM_INSTR_ERROR:
3994	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
3995	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
3996	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
3997	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
3998	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3999	case VMX_VMCS32_RO_IDT_INFO:
4000	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4001	case VMX_VMCS_GUEST_CR3:
4002	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4003	return true;
4004	}
4005	return false;
4006	}
4007
4008	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4009	{
4010	switch (idxField)
4011	{
4012	case VMX_VMCS_GUEST_LDTR_BASE:
4013	case VMX_VMCS_GUEST_TR_BASE:
4014	case VMX_VMCS_GUEST_GDTR_BASE:
4015	case VMX_VMCS_GUEST_IDTR_BASE:
4016	case VMX_VMCS_GUEST_SYSENTER_EIP:
4017	case VMX_VMCS_GUEST_SYSENTER_ESP:
4018	case VMX_VMCS_GUEST_CR0:
4019	case VMX_VMCS_GUEST_CR4:
4020	case VMX_VMCS_GUEST_CR3:
4021	case VMX_VMCS_GUEST_DR7:
4022	case VMX_VMCS_GUEST_RIP:
4023	case VMX_VMCS_GUEST_RSP:
4024	case VMX_VMCS_GUEST_CS_BASE:
4025	case VMX_VMCS_GUEST_DS_BASE:
4026	case VMX_VMCS_GUEST_ES_BASE:
4027	case VMX_VMCS_GUEST_FS_BASE:
4028	case VMX_VMCS_GUEST_GS_BASE:
4029	case VMX_VMCS_GUEST_SS_BASE:
4030	return true;
4031	}
4032	return false;
4033	}
4034	#endif /* VBOX_STRICT */
4035
4036	/**
4037	* Executes the specified handler in 64-bit mode.
4038	*
4039	* @returns VBox status code.
4040	* @param pVM Pointer to the VM.
4041	* @param pVCpu Pointer to the VMCPU.
4042	* @param pCtx Pointer to the guest CPU context.
4043	* @param pfnHandler Pointer to the RC handler function.
4044	* @param cbParam Number of parameters.
4045	* @param paParam Array of 32-bit parameters.
4046	*/
4047	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4048	uint32_t *paParam)
4049	{
4050	int rc, rc2;
4051	PHMGLOBLCPUINFO pCpu;
4052	RTHCPHYS HCPhysCpuPage;
4053	RTHCUINTREG uOldEFlags;
4054
4055	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4056	Assert(pfnHandler);
4057	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4058	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4059
4060	#ifdef VBOX_STRICT
4061	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4062	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4063
4064	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4065	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4066	#endif
4067
4068	/* Disable interrupts. */
4069	uOldEFlags = ASMIntDisableFlags();
4070
4071	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4072	RTCPUID idHostCpu = RTMpCpuId();
4073	CPUMR0SetLApic(pVM, idHostCpu);
4074	#endif
4075
4076	pCpu = HMR0GetCurrentCpu();
4077	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4078
4079	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4080	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4081
4082	/* Leave VMX Root Mode. */
4083	VMXDisable();
4084
4085	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4086
4087	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4088	CPUMSetHyperEIP(pVCpu, pfnHandler);
4089	for (int i = (int)cbParam - 1; i >= 0; i--)
4090	CPUMPushHyper(pVCpu, paParam[i]);
4091
4092	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4093
4094	/* Call the switcher. */
4095	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4096	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4097
4098	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4099	/* Make sure the VMX instructions don't cause #UD faults. */
4100	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4101
4102	/* Re-enter VMX Root Mode */
4103	rc2 = VMXEnable(HCPhysCpuPage);
4104	if (RT_FAILURE(rc2))
4105	{
4106	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4107	ASMSetFlags(uOldEFlags);
4108	return rc2;
4109	}
4110
4111	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4112	AssertRC(rc2);
4113	Assert(!(ASMGetFlags() & X86_EFL_IF));
4114	ASMSetFlags(uOldEFlags);
4115	return rc;
4116	}
4117
4118
4119	/**
4120	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4121	* supporting 64-bit guests.
4122	*
4123	* @returns VBox status code.
4124	* @param fResume Whether to VMLAUNCH or VMRESUME.
4125	* @param pCtx Pointer to the guest-CPU context.
4126	* @param pCache Pointer to the VMCS cache.
4127	* @param pVM Pointer to the VM.
4128	* @param pVCpu Pointer to the VMCPU.
4129	*/
4130	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4131	{
4132	uint32_t aParam[6];
4133	PHMGLOBLCPUINFO pCpu = NULL;
4134	RTHCPHYS HCPhysCpuPage = 0;
4135	int rc = VERR_INTERNAL_ERROR_5;
4136
4137	pCpu = HMR0GetCurrentCpu();
4138	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4139
4140	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4141	pCache->uPos = 1;
4142	pCache->interPD = PGMGetInterPaeCR3(pVM);
4143	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4144	#endif
4145
4146	#ifdef DEBUG
4147	pCache->TestIn.HCPhysCpuPage= 0;
4148	pCache->TestIn.HCPhysVmcs = 0;
4149	pCache->TestIn.pCache = 0;
4150	pCache->TestOut.HCPhysVmcs = 0;
4151	pCache->TestOut.pCache = 0;
4152	pCache->TestOut.pCtx = 0;
4153	pCache->TestOut.eflags = 0;
4154	#endif
4155
4156	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4157	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4158	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4159	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4160	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4161	aParam[5] = 0;
4162
4163	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4164	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4165	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4166	#endif
4167	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4168
4169	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4170	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4171	Assert(pCtx->dr[4] == 10);
4172	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4173	#endif
4174
4175	#ifdef DEBUG
4176	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4177	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4178	pVCpu->hm.s.vmx.HCPhysVmcs));
4179	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4180	pCache->TestOut.HCPhysVmcs));
4181	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4182	pCache->TestOut.pCache));
4183	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4184	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4185	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4186	pCache->TestOut.pCtx));
4187	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4188	#endif
4189	return rc;
4190	}
4191
4192
4193	/**
4194	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4195	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4196	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4197	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4198	*
4199	* @returns VBox status code.
4200	* @param pVM Pointer to the VM.
4201	* @param pVCpu Pointer to the VMCPU.
4202	*/
4203	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4204	{
4205	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4206	{ \
4207	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4208	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4209	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4210	}
4211
4212	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4213	{ \
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4218	}
4219
4220	AssertPtr(pVM);
4221	AssertPtr(pVCpu);
4222	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4223
4224	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4225	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4226	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4227	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4228	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4229	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4230	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4231	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4232	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4233
4234	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4235	#if 0
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4245	#endif
4246
4247	/* 32-bit guest-state fields. */
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4252	/* Unused 32-bit guest-state fields. */
4253	#if 0
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4255	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4256	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4257	#endif
4258
4259	/* Natural width guest-state fields. */
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4262	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4276	/* Unused natural width guest-state fields. */
4277	#if 0
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4280	#endif
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4283
4284	if (pVM->hm.s.fNestedPaging)
4285	{
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4288	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4289	}
4290	else
4291	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4292
4293	#undef VMXLOCAL_INIT_VMCS_SELREG
4294	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4295	return VINF_SUCCESS;
4296	}
4297
4298
4299	/**
4300	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4301	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4302	* darwin, running 64-bit guests).
4303	*
4304	* @returns VBox status code.
4305	* @param pVCpu Pointer to the VMCPU.
4306	* @param idxField The VMCS field encoding.
4307	* @param u64Val 16, 32 or 64 bits value.
4308	*/
4309	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4310	{
4311	int rc;
4312	switch (idxField)
4313	{
4314	/*
4315	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4316	*/
4317	/* 64-bit Control fields. */
4318	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4319	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4320	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4321	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4322	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4323	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4324	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4325	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4326	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4327	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4328	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4329	case VMX_VMCS64_CTRL_EPTP_FULL:
4330	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4331	/* 64-bit Read-only data fields. */
4332	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4333	/* 64-bit Guest-state fields. */
4334	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4335	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4336	case VMX_VMCS64_GUEST_PAT_FULL:
4337	case VMX_VMCS64_GUEST_EFER_FULL:
4338	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4339	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4340	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4341	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4342	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4343	/* 64-bit Host-state fields. */
4344	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4345	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4346	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4347	{
4348	rc = VMXWriteVmcs32(idxField, u64Val);
4349	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4350	break;
4351	}
4352
4353	/*
4354	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4355	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4356	*/
4357	/* Natural-width Guest-state fields. */
4358	case VMX_VMCS_GUEST_CR0:
4359	case VMX_VMCS_GUEST_CR3:
4360	case VMX_VMCS_GUEST_CR4:
4361	case VMX_VMCS_GUEST_ES_BASE:
4362	case VMX_VMCS_GUEST_CS_BASE:
4363	case VMX_VMCS_GUEST_SS_BASE:
4364	case VMX_VMCS_GUEST_DS_BASE:
4365	case VMX_VMCS_GUEST_FS_BASE:
4366	case VMX_VMCS_GUEST_GS_BASE:
4367	case VMX_VMCS_GUEST_LDTR_BASE:
4368	case VMX_VMCS_GUEST_TR_BASE:
4369	case VMX_VMCS_GUEST_GDTR_BASE:
4370	case VMX_VMCS_GUEST_IDTR_BASE:
4371	case VMX_VMCS_GUEST_DR7:
4372	case VMX_VMCS_GUEST_RSP:
4373	case VMX_VMCS_GUEST_RIP:
4374	case VMX_VMCS_GUEST_RFLAGS:
4375	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4376	case VMX_VMCS_GUEST_SYSENTER_ESP:
4377	case VMX_VMCS_GUEST_SYSENTER_EIP:
4378	{
4379	if ((u64Val >> 32ULL) == 0)
4380	{
4381	/* If this field is 64-bit, VT-x will zero out the top bits. */
4382	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4383	}
4384	else
4385	{
4386	/* Assert that only the 32->64 switcher case should ever come here. */
4387	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4388	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4389	}
4390	break;
4391	}
4392
4393	default:
4394	{
4395	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4396	rc = VERR_INVALID_PARAMETER;
4397	break;
4398	}
4399	}
4400	AssertRCReturn(rc, rc);
4401	return rc;
4402	}
4403
4404
4405	/**
4406	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4407	* hosts (except darwin) for 64-bit guests.
4408	*
4409	* @param pVCpu Pointer to the VMCPU.
4410	* @param idxField The VMCS field encoding.
4411	* @param u64Val 16, 32 or 64 bits value.
4412	*/
4413	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4414	{
4415	AssertPtr(pVCpu);
4416	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4417
4418	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4419	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4420
4421	/* Make sure there are no duplicates. */
4422	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4423	{
4424	if (pCache->Write.aField[i] == idxField)
4425	{
4426	pCache->Write.aFieldVal[i] = u64Val;
4427	return VINF_SUCCESS;
4428	}
4429	}
4430
4431	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4432	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4433	pCache->Write.cValidEntries++;
4434	return VINF_SUCCESS;
4435	}
4436
4437
4438	/**
4439	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4440	*
4441	* @param pVCpu Pointer to the VMCPU.
4442	* @param pCache Pointer to the VMCS cache.
4443	*
4444	* @remarks No-long-jump zone!!!
4445	*/
4446	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4447	{
4448	AssertPtr(pCache);
4449	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4450	{
4451	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4452	AssertRC(rc);
4453	}
4454	pCache->Write.cValidEntries = 0;
4455	}
4456
4457
4458	/**
4459	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4460	*
4461	* @param pVCpu Pointer to the VMCPU.
4462	* @param pCache Pointer to the VMCS cache.
4463	*
4464	* @remarks No-long-jump zone!!!
4465	*/
4466	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4467	{
4468	AssertPtr(pCache);
4469	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4470	{
4471	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4472	AssertRC(rc);
4473	}
4474	}
4475	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4476
4477
4478	/**
4479	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4480	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4481	* timer.
4482	*
4483	* @returns VBox status code.
4484	* @param pVM Pointer to the VM.
4485	* @param pVCpu Pointer to the VMCPU.
4486	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4487	* out-of-sync. Make sure to update the required fields
4488	* before using them.
4489	* @remarks No-long-jump zone!!!
4490	*/
4491	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4492	{
4493	int rc = VERR_INTERNAL_ERROR_5;
4494	bool fOffsettedTsc = false;
4495	if (pVM->hm.s.vmx.fUsePreemptTimer)
4496	{
4497	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4498
4499	/* Make sure the returned values have sane upper and lower boundaries. */
4500	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4501	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4502	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4503	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4504
4505	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4506	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4507	}
4508	else
4509	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4510
4511	if (fOffsettedTsc)
4512	{
4513	uint64_t u64CurTSC = ASMReadTSC();
4514	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4515	{
4516	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4517	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4518
4519	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4520	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4521	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4522	}
4523	else
4524	{
4525	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4526	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4527	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4528	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4529	}
4530	}
4531	else
4532	{
4533	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4534	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4535	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4536	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4537	}
4538	}
4539
4540
4541	/**
4542	* Determines if an exception is a benign exception. Benign exceptions
4543	* are ones which cannot cause double-faults.
4544	*
4545	* @returns true if the exception is benign, false otherwise.
4546	* @param uVector The exception vector.
4547	*/
4548	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4549	{
4550	switch (uVector)
4551	{
4552	case X86_XCPT_DB:
4553	case X86_XCPT_NMI:
4554	case X86_XCPT_BP:
4555	case X86_XCPT_OF:
4556	case X86_XCPT_BR:
4557	case X86_XCPT_UD:
4558	case X86_XCPT_NM:
4559	case X86_XCPT_CO_SEG_OVERRUN:
4560	case X86_XCPT_MF:
4561	case X86_XCPT_AC:
4562	case X86_XCPT_MC:
4563	case X86_XCPT_XF:
4564	return true;
4565	default:
4566	return false;
4567	}
4568	}
4569
4570
4571	/**
4572	* Determines if an exception is a contributory exception. Contributory
4573	* exceptions are ones which can cause double-faults.
4574	*
4575	* @returns true if the exception is contributory, false otherwise.
4576	* @param uVector The exception vector.
4577	*/
4578	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4579	{
4580	switch (uVector)
4581	{
4582	case X86_XCPT_GP:
4583	case X86_XCPT_SS:
4584	case X86_XCPT_NP:
4585	case X86_XCPT_TS:
4586	case X86_XCPT_DE:
4587	return true;
4588	default:
4589	return false;
4590	}
4591	return false;
4592	}
4593
4594
4595	/**
4596	* Determines if we are intercepting any contributory exceptions.
4597	*
4598	* @returns true if we are intercepting any contributory exception, false
4599	* otherwise.
4600	* @param pVCpu Pointer to the VMCPU.
4601	*/
4602	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4603	{
4604	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4605	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4606	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4607	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4608	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4609	{
4610	return true;
4611	}
4612	return false;
4613	}
4614
4615
4616	/**
4617	* Handle a condition that occurred while delivering an event through the guest
4618	* IDT.
4619	*
4620	* @returns VBox status code (informational error codes included).
4621	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4622	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4623	* continue execution of the guest which will delivery the #DF.
4624	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4625	*
4626	* @param pVM Pointer to the VM.
4627	* @param pVCpu Pointer to the VMCPU.
4628	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4629	* out-of-sync. Make sure to update the required fields
4630	* before using them.
4631	* @param pVmxTransient Pointer to the VMX transient structure.
4632	*
4633	* @remarks No-long-jump zone!!!
4634	*/
4635	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4636	{
4637	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4638	AssertRCReturn(rc, rc);
4639	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4640	{
4641	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4642	AssertRCReturn(rc, rc);
4643
4644	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4645	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4646	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4647
4648	typedef enum
4649	{
4650	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4651	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4652	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4653	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4654	} VMXREFLECTXCPT;
4655
4656	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4657	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4658	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4659	{
4660	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4661	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4662	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4663	&& uExitVector == X86_XCPT_PF))
4664	{
4665	enmReflect = VMXREFLECTXCPT_XCPT;
4666	}
4667	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4668	&& uIdtVector == X86_XCPT_PF
4669	&& uExitVector == X86_XCPT_PF)
4670	{
4671	pVmxTransient->fVectoringPF = true;
4672	}
4673	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4674	&& hmR0VmxIsContributoryXcpt(uExitVector))
4675	{
4676	enmReflect = VMXREFLECTXCPT_DF;
4677	}
4678	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4679	&& uIdtVector == X86_XCPT_PF
4680	&& hmR0VmxIsContributoryXcpt(uExitVector))
4681	{
4682	enmReflect = VMXREFLECTXCPT_DF;
4683	}
4684	else if (uIdtVector == X86_XCPT_DF)
4685	enmReflect = VMXREFLECTXCPT_TF;
4686	}
4687	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4688	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4689	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4690	{
4691	/*
4692	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4693	* (whatever they are) as they reoccur when restarting the instruction.
4694	*/
4695	enmReflect = VMXREFLECTXCPT_XCPT;
4696	}
4697
4698	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4699	switch (enmReflect)
4700	{
4701	case VMXREFLECTXCPT_XCPT:
4702	{
4703	Assert(!pVCpu->hm.s.Event.fPending);
4704	pVCpu->hm.s.Event.fPending = true;
4705	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4706	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4707	{
4708	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4709	AssertRCReturn(rc, rc);
4710	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4711	}
4712	else
4713	pVCpu->hm.s.Event.u32ErrCode = 0;
4714	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4715	break;
4716	}
4717
4718	case VMXREFLECTXCPT_DF:
4719	{
4720	uint32_t u32IntrInfo;
4721	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4722	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4723	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4724
4725	Assert(!pVCpu->hm.s.Event.fPending);
4726	pVCpu->hm.s.Event.fPending = true;
4727	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4728	pVCpu->hm.s.Event.u32ErrCode = 0;
4729	rc = VINF_VMX_DOUBLE_FAULT;
4730	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4731	break;
4732	}
4733
4734	case VMXREFLECTXCPT_TF:
4735	{
4736	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4737	rc = VINF_EM_RESET;
4738	break;
4739	}
4740
4741	default: /* shut up gcc. */
4742	break;
4743	}
4744	}
4745	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4746	return rc;
4747	}
4748
4749
4750	/**
4751	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4752	*
4753	* @returns VBox status code.
4754	* @param pVM Pointer to the VM.
4755	* @param pVCpu Pointer to the VMCPU.
4756	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4757	* out-of-sync. Make sure to update the required fields
4758	* before using them.
4759	*
4760	* @remarks No-long-jump zone!!!
4761	*/
4762	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4763	{
4764	int rc = VINF_SUCCESS;
4765	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4766	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4767	{
4768	RTGCUINTREG uVal = 0;
4769	RTCCUINTREG uShadow = 0;
4770	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4771	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4772	AssertRCReturn(rc, rc);
4773	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4774	CPUMSetGuestCR0(pVCpu, uVal);
4775	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4776	}
4777	return rc;
4778	}
4779
4780
4781	/**
4782	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4783	*
4784	* @returns VBox status code.
4785	* @param pVM Pointer to the VM.
4786	* @param pVCpu Pointer to the VMCPU.
4787	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4788	* out-of-sync. Make sure to update the required fields
4789	* before using them.
4790	*
4791	* @remarks No-long-jump zone!!!
4792	*/
4793	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4794	{
4795	int rc = VINF_SUCCESS;
4796	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4797	{
4798	RTGCUINTREG uVal = 0;
4799	RTCCUINTREG uShadow = 0;
4800	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4801	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4802	AssertRCReturn(rc, rc);
4803	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4804	CPUMSetGuestCR4(pVCpu, uVal);
4805	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4806	}
4807	return rc;
4808	}
4809
4810
4811	/**
4812	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4813	*
4814	* @returns VBox status code.
4815	* @param pVM Pointer to the VM.
4816	* @param pVCpu Pointer to the VMCPU.
4817	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4818	* out-of-sync. Make sure to update the required fields
4819	* before using them.
4820	*
4821	* @remarks No-long-jump zone!!!
4822	*/
4823	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824	{
4825	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4826	return VINF_SUCCESS;
4827
4828	RTGCUINTREG uVal = 0;
4829	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4830	AssertRCReturn(rc, rc);
4831	pMixedCtx->rip = uVal;
4832	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4833	return rc;
4834	}
4835
4836
4837	/**
4838	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4839	*
4840	* @returns VBox status code.
4841	* @param pVM Pointer to the VM.
4842	* @param pVCpu Pointer to the VMCPU.
4843	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4844	* out-of-sync. Make sure to update the required fields
4845	* before using them.
4846	*
4847	* @remarks No-long-jump zone!!!
4848	*/
4849	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4850	{
4851	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4852	return VINF_SUCCESS;
4853
4854	RTGCUINTREG uVal = 0;
4855	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4856	AssertRCReturn(rc, rc);
4857	pMixedCtx->rsp = uVal;
4858	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4859	return rc;
4860	}
4861
4862
4863	/**
4864	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4865	*
4866	* @returns VBox status code.
4867	* @param pVM Pointer to the VM.
4868	* @param pVCpu Pointer to the VMCPU.
4869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4870	* out-of-sync. Make sure to update the required fields
4871	* before using them.
4872	*
4873	* @remarks No-long-jump zone!!!
4874	*/
4875	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4876	{
4877	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4878	return VINF_SUCCESS;
4879
4880	RTGCUINTREG uVal = 0;
4881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4882	AssertRCReturn(rc, rc);
4883	pMixedCtx->rflags.u64 = uVal;
4884
4885	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4886	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4887	{
4888	Assert(pVM->hm.s.vmx.pRealModeTSS);
4889	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4890	pMixedCtx->eflags.Bits.u1VM = 0;
4891	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4892	}
4893
4894	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4895	return rc;
4896	}
4897
4898
4899	/**
4900	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4901	* guest-CPU context.
4902	*/
4903	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4904	{
4905	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4906	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4907	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4908	return rc;
4909	}
4910
4911
4912	/**
4913	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4914	* from the guest-state area in the VMCS.
4915	*
4916	* @param pVM Pointer to the VM.
4917	* @param pVCpu Pointer to the VMCPU.
4918	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4919	* out-of-sync. Make sure to update the required fields
4920	* before using them.
4921	*
4922	* @remarks No-long-jump zone!!!
4923	*/
4924	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4925	{
4926	uint32_t uIntrState = 0;
4927	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4928	AssertRC(rc);
4929
4930	if (!uIntrState)
4931	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4932	else
4933	{
4934	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4935	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4936	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4937	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4938	AssertRC(rc);
4939	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4940	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4941	}
4942	}
4943
4944
4945	/**
4946	* Saves the guest's activity state.
4947	*
4948	* @returns VBox status code.
4949	* @param pVM Pointer to the VM.
4950	* @param pVCpu Pointer to the VMCPU.
4951	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4952	* out-of-sync. Make sure to update the required fields
4953	* before using them.
4954	*
4955	* @remarks No-long-jump zone!!!
4956	*/
4957	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4958	{
4959	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4960	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4961	return VINF_SUCCESS;
4962	}
4963
4964
4965	/**
4966	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4967	* the current VMCS into the guest-CPU context.
4968	*
4969	* @returns VBox status code.
4970	* @param pVM Pointer to the VM.
4971	* @param pVCpu Pointer to the VMCPU.
4972	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4973	* out-of-sync. Make sure to update the required fields
4974	* before using them.
4975	*
4976	* @remarks No-long-jump zone!!!
4977	*/
4978	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4979	{
4980	int rc = VINF_SUCCESS;
4981	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4982	{
4983	uint32_t u32Val = 0;
4984	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4985	pMixedCtx->SysEnter.cs = u32Val;
4986	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4987	}
4988
4989	RTGCUINTREG uGCVal = 0;
4990	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4991	{
4992	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4993	pMixedCtx->SysEnter.eip = uGCVal;
4994	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4995	}
4996	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4997	{
4998	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4999	pMixedCtx->SysEnter.esp = uGCVal;
5000	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5001	}
5002	return rc;
5003	}
5004
5005
5006	/**
5007	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5008	* context.
5009	*
5010	* @returns VBox status code.
5011	* @param pVM Pointer to the VM.
5012	* @param pVCpu Pointer to the VMCPU.
5013	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5014	* out-of-sync. Make sure to update the required fields
5015	* before using them.
5016	*
5017	* @remarks No-long-jump zone!!!
5018	*/
5019	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5020	{
5021	RTGCUINTREG uVal = 0;
5022	int rc = VINF_SUCCESS;
5023	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5024	{
5025	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5026	pMixedCtx->fs.u64Base = uVal;
5027	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5028	}
5029	return rc;
5030	}
5031
5032
5033	/**
5034	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5035	* context.
5036	*
5037	* @returns VBox status code.
5038	* @param pVM Pointer to the VM.
5039	* @param pVCpu Pointer to the VMCPU.
5040	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5041	* out-of-sync. Make sure to update the required fields
5042	* before using them.
5043	*
5044	* @remarks No-long-jump zone!!!
5045	*/
5046	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5047	{
5048	RTGCUINTREG uVal = 0;
5049	int rc = VINF_SUCCESS;
5050	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5051	{
5052	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5053	pMixedCtx->gs.u64Base = uVal;
5054	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5055	}
5056	return rc;
5057	}
5058
5059
5060	/**
5061	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5062	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5063	*
5064	* @returns VBox status code.
5065	* @param pVM Pointer to the VM.
5066	* @param pVCpu Pointer to the VMCPU.
5067	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5068	* out-of-sync. Make sure to update the required fields
5069	* before using them.
5070	*
5071	* @remarks No-long-jump zone!!!
5072	*/
5073	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5074	{
5075	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5076	return VINF_SUCCESS;
5077
5078	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5079	{
5080	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5081	pMsr += i;
5082	switch (pMsr->u32IndexMSR)
5083	{
5084	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5085	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5086	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5087	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5088	#if 0
5089	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5090	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5091	#endif
5092	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5093	default:
5094	{
5095	AssertFailed();
5096	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5097	}
5098	}
5099	}
5100	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5101	return VINF_SUCCESS;
5102	}
5103
5104
5105	/**
5106	* Saves the guest control registers from the current VMCS into the guest-CPU
5107	* context.
5108	*
5109	* @returns VBox status code.
5110	* @param pVM Pointer to the VM.
5111	* @param pVCpu Pointer to the VMCPU.
5112	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5113	* out-of-sync. Make sure to update the required fields
5114	* before using them.
5115	*
5116	* @remarks No-long-jump zone!!!
5117	*/
5118	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5119	{
5120	RTGCUINTREG uVal = 0;
5121	RTGCUINTREG uShadow = 0;
5122	int rc = VINF_SUCCESS;
5123
5124	/* Guest CR0. Guest FPU. */
5125	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5126
5127	/* Guest CR4. */
5128	rc \|= hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5129	AssertRCReturn(rc, rc);
5130
5131	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5132	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5133	{
5134	if ( pVM->hm.s.fNestedPaging
5135	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5136	{
5137	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5138	if (pMixedCtx->cr3 != uVal)
5139	{
5140	CPUMSetGuestCR3(pVCpu, uVal);
5141	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5142	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5143	}
5144
5145	/* We require EFER to check PAE mode. */
5146	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5147
5148	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5149	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5150	{
5151	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5152	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5153	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5154	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5155	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5156	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5157	}
5158	AssertRCReturn(rc, rc);
5159	}
5160	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5161	}
5162	return rc;
5163	}
5164
5165
5166	/**
5167	* Reads a guest segment register from the current VMCS into the guest-CPU
5168	* context.
5169	*
5170	* @returns VBox status code.
5171	* @param pVCpu Pointer to the VMCPU.
5172	* @param idxSel Index of the selector in the VMCS.
5173	* @param idxLimit Index of the segment limit in the VMCS.
5174	* @param idxBase Index of the segment base in the VMCS.
5175	* @param idxAccess Index of the access rights of the segment in the VMCS.
5176	* @param pSelReg Pointer to the segment selector.
5177	*
5178	* @remarks No-long-jump zone!!!
5179	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5180	* as that takes care of whether to read from the VMCS cache or not.
5181	*/
5182	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5183	PCPUMSELREG pSelReg)
5184	{
5185	uint32_t u32Val = 0;
5186	int rc = VMXReadVmcs32(idxSel, &u32Val);
5187	pSelReg->Sel = (uint16_t)u32Val;
5188	pSelReg->ValidSel = (uint16_t)u32Val;
5189	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5190
5191	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5192	pSelReg->u32Limit = u32Val;
5193
5194	RTGCUINTREG uGCVal = 0;
5195	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5196	pSelReg->u64Base = uGCVal;
5197
5198	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5199	pSelReg->Attr.u = u32Val;
5200	AssertRCReturn(rc, rc);
5201
5202	/*
5203	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5204	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5205	*/
5206	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5207	{
5208	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5209	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5210	}
5211	return rc;
5212	}
5213
5214
5215	/**
5216	* Saves the guest segment registers from the current VMCS into the guest-CPU
5217	* context.
5218	*
5219	* @returns VBox status code.
5220	* @param pVM Pointer to the VM.
5221	* @param pVCpu Pointer to the VMCPU.
5222	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5223	* out-of-sync. Make sure to update the required fields
5224	* before using them.
5225	*
5226	* @remarks No-long-jump zone!!!
5227	*/
5228	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5229	{
5230	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5231	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5232	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5233	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5234	#else
5235	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5236	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5237	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5238	#endif
5239
5240	int rc = VINF_SUCCESS;
5241
5242	/* Guest segment registers. */
5243	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5244	{
5245	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5246	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5247	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5248	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5249	rc \|= VMXLOCAL_READ_SEG(ES, es);
5250	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5251	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5252	AssertRCReturn(rc, rc);
5253
5254	/* Restore segment attributes for real-on-v86 mode hack. */
5255	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5256	{
5257	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5258	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5259	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5260	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5261	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5262	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5263	}
5264	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5265	}
5266
5267	/* Guest LDTR. */
5268	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5269	{
5270	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5271	AssertRCReturn(rc, rc);
5272	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5273	}
5274
5275	/* Guest GDTR. */
5276	RTGCUINTREG uGCVal = 0;
5277	uint32_t u32Val = 0;
5278	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5279	{
5280	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5281	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5282	pMixedCtx->gdtr.pGdt = uGCVal;
5283	pMixedCtx->gdtr.cbGdt = u32Val;
5284	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5285	}
5286
5287	/* Guest IDTR. */
5288	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5289	{
5290	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5291	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5292	pMixedCtx->idtr.pIdt = uGCVal;
5293	pMixedCtx->idtr.cbIdt = u32Val;
5294	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5295	}
5296
5297	/* Guest TR. */
5298	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5299	{
5300	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5301
5302	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5303	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5304	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5305	AssertRCReturn(rc, rc);
5306	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5307	}
5308	return rc;
5309	}
5310
5311
5312	/**
5313	* Saves the guest debug registers from the current VMCS into the guest-CPU
5314	* context.
5315	*
5316	* @returns VBox status code.
5317	* @param pVM Pointer to the VM.
5318	* @param pVCpu Pointer to the VMCPU.
5319	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5320	* out-of-sync. Make sure to update the required fields
5321	* before using them.
5322	*
5323	* @remarks No-long-jump zone!!!
5324	*/
5325	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5326	{
5327	int rc = VINF_SUCCESS;
5328	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5329	{
5330	RTGCUINTREG uVal;
5331	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5332	pMixedCtx->dr[7] = uVal;
5333
5334	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5335	}
5336	return rc;
5337	}
5338
5339
5340	/**
5341	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5342	*
5343	* @returns VBox status code.
5344	* @param pVM Pointer to the VM.
5345	* @param pVCpu Pointer to the VMCPU.
5346	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5347	* out-of-sync. Make sure to update the required fields
5348	* before using them.
5349	*
5350	* @remarks No-long-jump zone!!!
5351	*/
5352	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5353	{
5354	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5355	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5356	return VINF_SUCCESS;
5357	}
5358
5359
5360	/**
5361	* Saves the entire guest state from the currently active VMCS into the
5362	* guest-CPU context. This essentially VMREADs all guest-data.
5363	*
5364	* @returns VBox status code.
5365	* @param pVM Pointer to the VM.
5366	* @param pVCpu Pointer to the VMCPU.
5367	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5368	* out-of-sync. Make sure to update the required fields
5369	* before using them.
5370	*/
5371	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5372	{
5373	Assert(pVM);
5374	Assert(pVCpu);
5375	Assert(pMixedCtx);
5376
5377	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5378	return VINF_SUCCESS;
5379
5380	VMMRZCallRing3Disable(pVCpu);
5381
5382	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5384
5385	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5387
5388	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5389	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5390
5391	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5392	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5393
5394	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5395	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5396
5397	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5398	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5399
5400	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5401	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5402
5403	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5404	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5405
5406	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5407	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5408
5409	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5410	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5411
5412	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5413	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5414
5415	VMMRZCallRing3Enable(pVCpu);
5416	return rc;
5417	}
5418
5419
5420	/**
5421	* Check per-VM and per-VCPU force flag actions that require us to go back to
5422	* ring-3 for one reason or another.
5423	*
5424	* @returns VBox status code (information status code included).
5425	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5426	* ring-3.
5427	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5428	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5429	* interrupts)
5430	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5431	* all EMTs to be in ring-3.
5432	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5433	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5434	* to the EM loop.
5435	*
5436	* @param pVM Pointer to the VM.
5437	* @param pVCpu Pointer to the VMCPU.
5438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5439	* out-of-sync. Make sure to update the required fields
5440	* before using them.
5441	*/
5442	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5443	{
5444	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5445
5446	int rc = VERR_INTERNAL_ERROR_5;
5447	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5448	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5449	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5450	{
5451	/* We need the control registers now, make sure the guest-CPU context is updated. */
5452	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5453	AssertRCReturn(rc, rc);
5454
5455	/* Pending HM CR3 sync. */
5456	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5457	{
5458	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5459	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5460	}
5461	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5462	{
5463	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5464	AssertRC(rc);
5465	}
5466
5467	/* Pending PGM C3 sync. */
5468	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5469	{
5470	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5471	if (rc != VINF_SUCCESS)
5472	{
5473	AssertRC(rc);
5474	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5475	return rc;
5476	}
5477	}
5478
5479	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5480	/* -XXX- what was that about single stepping? */
5481	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5482	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5483	{
5484	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5485	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5486	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5487	return rc;
5488	}
5489
5490	/* Pending VM request packets, such as hardware interrupts. */
5491	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5492	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5493	{
5494	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5495	return VINF_EM_PENDING_REQUEST;
5496	}
5497
5498	/* Pending PGM pool flushes. */
5499	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5500	{
5501	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5502	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5503	}
5504
5505	/* Pending DMA requests. */
5506	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5507	{
5508	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5509	return VINF_EM_RAW_TO_R3;
5510	}
5511	}
5512
5513	/* Paranoia. */
5514	Assert(rc != VERR_EM_INTERPRETER);
5515	return VINF_SUCCESS;
5516	}
5517
5518
5519	/**
5520	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5521	* VT-x to execute any instruction.
5522	*
5523	* @param pvCpu Pointer to the VMCPU.
5524	*/
5525	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5526	{
5527	if (pVCpu->hm.s.Event.fPending)
5528	{
5529	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5530	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5531	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5532	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5533
5534	/* If a trap was already pending, we did something wrong! */
5535	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5536
5537	/* A page-fault exception during a page-fault would become a double-fault. */
5538	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5539	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5540
5541	TRPMEVENT enmTrapType;
5542	switch (uVectorType)
5543	{
5544	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5545	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5546	enmTrapType = TRPM_HARDWARE_INT;
5547	break;
5548	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5549	enmTrapType = TRPM_SOFTWARE_INT;
5550	break;
5551	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5552	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5553	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5554	enmTrapType = TRPM_TRAP;
5555	break;
5556	default:
5557	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5558	enmTrapType = TRPM_32BIT_HACK;
5559	break;
5560	}
5561	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5562	AssertRC(rc);
5563	if (fErrorCodeValid)
5564	TRPMSetErrorCode(pVCpu, uErrorCode);
5565
5566	/* Clear the VT-x state bits now that TRPM has the information. */
5567	pVCpu->hm.s.Event.fPending = false;
5568	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5569	AssertRC(rc);
5570	}
5571	}
5572
5573
5574	/**
5575	* Does the necessary state syncing before doing a longjmp to ring-3.
5576	*
5577	* @param pVM Pointer to the VM.
5578	* @param pVCpu Pointer to the VMCPU.
5579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5580	* out-of-sync. Make sure to update the required fields
5581	* before using them.
5582	* @param rcExit The reason for exiting to ring-3. Can be
5583	* VINF_VMM_UNKNOWN_RING3_CALL.
5584	*
5585	* @remarks No-long-jmp zone!!!
5586	*/
5587	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5588	{
5589	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5590
5591	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5592	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5593	AssertRC(rc);
5594
5595	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5596	if (CPUMIsGuestFPUStateActive(pVCpu))
5597	{
5598	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5599	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5600	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5601	}
5602
5603	/* Restore debug registers if necessary and resync on next R0 reentry. */
5604	if (CPUMIsGuestDebugStateActive(pVCpu))
5605	{
5606	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5607	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5608	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5609	}
5610	else if (CPUMIsHyperDebugStateActive(pVCpu))
5611	{
5612	CPUMR0LoadHostDebugState(pVM, pVCpu);
5613	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5614	}
5615
5616	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5617	}
5618
5619
5620	/**
5621	* An action requires us to go back to ring-3. This function does the necessary
5622	* steps before we can safely return to ring-3. This is not the same as longjmps
5623	* to ring-3, this is voluntary.
5624	*
5625	* @param pVM Pointer to the VM.
5626	* @param pVCpu Pointer to the VMCPU.
5627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5628	* out-of-sync. Make sure to update the required fields
5629	* before using them.
5630	* @param rcExit The reason for exiting to ring-3. Can be
5631	* VINF_VMM_UNKNOWN_RING3_CALL.
5632	*/
5633	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5634	{
5635	Assert(pVM);
5636	Assert(pVCpu);
5637	Assert(pMixedCtx);
5638	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5639
5640	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5641	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5642	return;
5643
5644	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5645	VMMRZCallRing3Disable(pVCpu);
5646	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5647
5648	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5649	hmR0VmxUpdateTRPMTrap(pVCpu);
5650
5651	/* Sync. the guest state. */
5652	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5653	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5654
5655	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5656	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5657	\| CPUM_CHANGED_LDTR
5658	\| CPUM_CHANGED_GDTR
5659	\| CPUM_CHANGED_IDTR
5660	\| CPUM_CHANGED_TR
5661	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5662
5663	/* On our way back from ring-3 the following needs to be done. */
5664	/** @todo This can change with preemption hooks. */
5665	if (rcExit == VINF_EM_RAW_INTERRUPT)
5666	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5667	else
5668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5669
5670	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5671	VMMRZCallRing3Enable(pVCpu);
5672	}
5673
5674
5675	/**
5676	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5677	* longjump to ring-3 and possibly get preempted.
5678	*
5679	* @param pVCpu Pointer to the VMCPU.
5680	* @param enmOperation The operation causing the ring-3 longjump.
5681	* @param pvUser The user argument (pointer to the possibly
5682	* out-of-date guest-CPU context).
5683	*
5684	* @remarks Must never be called with @a enmOperation ==
5685	* VMMCALLRING3_VM_R0_ASSERTION.
5686	*/
5687	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5688	{
5689	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5690	Assert(pVCpu);
5691	Assert(pvUser);
5692	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5693	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5694
5695	VMMRZCallRing3Disable(pVCpu);
5696	Log(("hmR0VmxLongJmpToRing3\n"));
5697	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5698	VMMRZCallRing3Enable(pVCpu);
5699	}
5700
5701
5702	/**
5703	* Injects any pending TRPM trap into the VM by updating the VMCS.
5704	*
5705	* @returns VBox status code (informational status code included).
5706	* @param pVM Pointer to the VM.
5707	* @param pVCpu Pointer to the VMCPU.
5708	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5709	* out-of-sync. Make sure to update the required fields
5710	* before using them.
5711	*/
5712	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5713	{
5714	if (!TRPMHasTrap(pVCpu))
5715	return VINF_SUCCESS;
5716
5717	uint8_t u8Vector = 0;
5718	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5719	RTGCUINT uErrCode = 0;
5720
5721	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5722	AssertRCReturn(rc, rc);
5723	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5724
5725	rc = TRPMResetTrap(pVCpu);
5726	AssertRCReturn(rc, rc);
5727
5728	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5729	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5730	if (enmTrpmEvent == TRPM_TRAP)
5731	{
5732	switch (u8Vector)
5733	{
5734	case X86_XCPT_BP:
5735	case X86_XCPT_OF:
5736	{
5737	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5738	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5739	break;
5740	}
5741
5742	case X86_XCPT_DF:
5743	case X86_XCPT_TS:
5744	case X86_XCPT_NP:
5745	case X86_XCPT_SS:
5746	case X86_XCPT_GP:
5747	case X86_XCPT_PF:
5748	case X86_XCPT_AC:
5749	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5750	them which VT-x/VMM pushes to the guest stack. */
5751	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5752	/* no break! */
5753	default:
5754	{
5755	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5756	break;
5757	}
5758	}
5759	}
5760	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5761	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5762	else
5763	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5764
5765	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5766	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5767	}
5768
5769
5770	/**
5771	* Checks if there are any pending guest interrupts to be delivered and injects
5772	* them into the VM by updating the VMCS.
5773	*
5774	* @returns VBox status code (informational status codes included).
5775	* @param pVM Pointer to the VM.
5776	* @param pVCpu Pointer to the VMCPU.
5777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778	* out-of-sync. Make sure to update the required fields
5779	* before using them.
5780	*
5781	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5782	*/
5783	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5784	{
5785	/* First inject any pending HM interrupts. */
5786	if (pVCpu->hm.s.Event.fPending)
5787	{
5788	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5789	pVCpu->hm.s.Event.u32ErrCode);
5790	AssertRCReturn(rc, rc);
5791	pVCpu->hm.s.Event.fPending = false;
5792	return rc;
5793	}
5794
5795	/** @todo SMI. SMIs take priority over NMIs. */
5796
5797	/* NMI. NMIs take priority over regular interrupts . */
5798	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5799	{
5800	/* Construct an NMI interrupt and inject it into the VMCS. */
5801	RTGCUINTPTR uIntrInfo;
5802	uIntrInfo = X86_XCPT_NMI;
5803	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5804	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5805	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5806	AssertRCReturn(rc, rc);
5807	return rc;
5808	}
5809
5810	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5811	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5812	AssertRCReturn(rc, rc);
5813
5814	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5815	if (!TRPMHasTrap(pVCpu))
5816	{
5817	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5818	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5819	{
5820	/*
5821	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5822	* the PDM interrupt into a TRPM event and inject it.
5823	*/
5824	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5825	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5826	{
5827	uint8_t u8Interrupt = 0;
5828	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5829	if (RT_SUCCESS(rc))
5830	{
5831	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5832	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5833	AssertRCReturn(rc, rc);
5834	}
5835	else
5836	{
5837	/** @todo Does this actually happen? If not turn it into an assertion. */
5838	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5839	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5840	}
5841	}
5842	else if ( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT)
5843	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5844	{
5845	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5846	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5847	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5848	AssertRCReturn(rc, rc);
5849	}
5850	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5851	}
5852	}
5853
5854	/* If interrupts can be delivered, inject it into the VM. */
5855	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5856	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5857	&& TRPMHasTrap(pVCpu))
5858	{
5859	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5860	AssertRCReturn(rc, rc);
5861	}
5862	return rc;
5863	}
5864
5865	/**
5866	* Injects an invalid-opcode (#UD) exception into the VM.
5867	*
5868	* @returns VBox status code (informational status code included).
5869	* @param pVM Pointer to the VM.
5870	* @param pVCpu Pointer to the VMCPU.
5871	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5872	* out-of-sync. Make sure to update the required fields
5873	* before using them.
5874	*/
5875	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5876	{
5877	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5878	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5879	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5880	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5881	}
5882
5883
5884	/**
5885	* Injects a double-fault (#DF) exception into the VM.
5886	*
5887	* @returns VBox status code (informational status code included).
5888	* @param pVM Pointer to the VM.
5889	* @param pVCpu Pointer to the VMCPU.
5890	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5891	* out-of-sync. Make sure to update the required fields
5892	* before using them.
5893	*/
5894	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5895	{
5896	/* Inject the double-fault. */
5897	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5898	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5899	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5900	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5901	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5902	}
5903
5904
5905	/**
5906	* Injects a debug (#DB) exception into the VM.
5907	*
5908	* @returns VBox status code (informational status code included).
5909	* @param pVM Pointer to the VM.
5910	* @param pVCpu Pointer to the VMCPU.
5911	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5912	* out-of-sync. Make sure to update the required fields
5913	* before using them.
5914	*/
5915	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5916	{
5917	/* Inject the debug-exception. */
5918	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5919	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5920	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5921	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5922	}
5923
5924
5925	/**
5926	* Injects a overflow (#OF) exception into the VM.
5927	*
5928	* @returns VBox status code (informational status code included).
5929	* @param pVM Pointer to the VM.
5930	* @param pVCpu Pointer to the VMCPU.
5931	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5932	* out-of-sync. Make sure to update the required fields
5933	* before using them.
5934	* @param cbInstr The value of RIP that is to be pushed on the guest
5935	* stack.
5936	*/
5937	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5938	{
5939	/* Inject the overflow exception. */
5940	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5941	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5942	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5943	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5944	}
5945
5946
5947	/**
5948	* Injects a general-protection (#GP) fault into the VM.
5949	*
5950	* @returns VBox status code (informational status code included).
5951	* @param pVM Pointer to the VM.
5952	* @param pVCpu Pointer to the VMCPU.
5953	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5954	* out-of-sync. Make sure to update the required fields
5955	* before using them.
5956	* @param u32ErrorCode The error code associated with the #GP.
5957	*/
5958	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5959	{
5960	/* Inject the general-protection fault. */
5961	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5962	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5963	if (fErrorCodeValid)
5964	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5965	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5966	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5967	}
5968
5969
5970	/**
5971	* Injects a software interrupt (INTn) into the VM.
5972	*
5973	* @returns VBox status code (informational status code included).
5974	* @param pVM Pointer to the VM.
5975	* @param pVCpu Pointer to the VMCPU.
5976	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5977	* out-of-sync. Make sure to update the required fields
5978	* before using them.
5979	* @param uVector The software interrupt vector number.
5980	* @param cbInstr The value of RIP that is to be pushed on the guest
5981	* stack.
5982	*/
5983	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5984	{
5985	/* Inject the INTn. */
5986	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5987	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5988	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5989	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5990	}
5991
5992
5993	/**
5994	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5995	* stack.
5996	*
5997	* @returns VBox status code (information status code included).
5998	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5999	* @param pVM Pointer to the VM.
6000	* @param pMixedCtx Pointer to the guest-CPU context.
6001	* @param uValue The value to push to the guest stack.
6002	*/
6003	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6004	{
6005	/*
6006	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6007	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6008	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6009	*/
6010	if (pMixedCtx->sp == 1)
6011	return VINF_EM_RESET;
6012	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6013	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6014	AssertRCReturn(rc, rc);
6015	return rc;
6016	}
6017
6018
6019	/**
6020	* Injects an event into the guest upon VM-entry by updating the relevant fields
6021	* in the VM-entry area in the VMCS.
6022	*
6023	* @returns VBox status code (informational error codes included).
6024	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6025	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6026	*
6027	* @param pVM Pointer to the VM.
6028	* @param pVCpu Pointer to the VMCPU.
6029	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6030	* out-of-sync. Make sure to update the required fields
6031	* before using them.
6032	* @param u64IntrInfo The VM-entry interruption-information field.
6033	* @param cbInstr The VM-entry instruction length in bytes (for software
6034	* interrupts, exceptions and privileged software
6035	* exceptions).
6036	* @param u32ErrCode The VM-entry exception error code.
6037	*
6038	* @remarks No-long-jump zone!!!
6039	*/
6040	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6041	uint32_t u32ErrCode)
6042	{
6043	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6044	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6045	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6046
6047	/* We require CR0 to check if the guest is in real-mode. */
6048	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6049	AssertRCReturn(rc, rc);
6050
6051	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6052	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6053
6054	/*
6055	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6056	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6057	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6058	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6059	*/
6060	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6061	{
6062	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6063	{
6064	Assert(PDMVmmDevHeapIsEnabled(pVM));
6065	Assert(pVM->hm.s.vmx.pRealModeTSS);
6066
6067	/* Save the required guest state bits from the VMCS. */
6068	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6069	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6070	AssertRCReturn(rc, rc);
6071
6072	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6073	const size_t cbIdtEntry = 4;
6074	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6075	{
6076	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6077	if (uVector == X86_XCPT_DF)
6078	return VINF_EM_RESET;
6079	else if (uVector == X86_XCPT_GP)
6080	{
6081	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6082	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6083	}
6084
6085	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6086	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6087	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6088	}
6089
6090	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6091	uint16_t uGuestIp = pMixedCtx->ip;
6092	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6093	{
6094	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6095	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6096	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6097	}
6098	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6099	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6100
6101	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6102	uint16_t offIdtEntry = 0;
6103	RTSEL selIdtEntry = 0;
6104	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6105	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6106	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6107	AssertRCReturn(rc, rc);
6108
6109	/* Construct the stack frame for the interrupt/exception handler. */
6110	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6111	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6112	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6113	AssertRCReturn(rc, rc);
6114
6115	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6116	if (rc == VINF_SUCCESS)
6117	{
6118	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6119	pMixedCtx->rip = offIdtEntry;
6120	pMixedCtx->cs.Sel = selIdtEntry;
6121	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6122	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6123	\| HM_CHANGED_GUEST_RIP
6124	\| HM_CHANGED_GUEST_RFLAGS
6125	\| HM_CHANGED_GUEST_RSP;
6126	}
6127	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6128	return rc;
6129	}
6130	else
6131	{
6132	/*
6133	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6134	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6135	*/
6136	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6137	}
6138	}
6139
6140	/* Add the valid bit, maybe the caller was lazy. */
6141	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6142
6143	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6144	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6145	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6146
6147	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6148	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6149	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6150	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6151	AssertRCReturn(rc, rc);
6152	return rc;
6153	}
6154
6155
6156	/**
6157	* Enters the VT-x session.
6158	*
6159	* @returns VBox status code.
6160	* @param pVM Pointer to the VM.
6161	* @param pVCpu Pointer to the VMCPU.
6162	* @param pCpu Pointer to the CPU info struct.
6163	*/
6164	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6165	{
6166	AssertPtr(pVM);
6167	AssertPtr(pVCpu);
6168	Assert(pVM->hm.s.vmx.fSupported);
6169	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6170	NOREF(pCpu);
6171
6172	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6173
6174	/* Make sure we're in VMX root mode. */
6175	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6176	if (!(u32HostCR4 & X86_CR4_VMXE))
6177	{
6178	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6179	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6180	}
6181
6182	/* Load the active VMCS as the current one. */
6183	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6184	if (RT_FAILURE(rc))
6185	return rc;
6186
6187	/** @todo this will change with preemption hooks where can can VMRESUME as long
6188	* as we're no preempted. */
6189	pVCpu->hm.s.fResumeVM = false;
6190	return VINF_SUCCESS;
6191	}
6192
6193
6194	/**
6195	* Leaves the VT-x session.
6196	*
6197	* @returns VBox status code.
6198	* @param pVM Pointer to the VM.
6199	* @param pVCpu Pointer to the VMCPU.
6200	* @param pCtx Pointer to the guest-CPU context.
6201	*/
6202	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6203	{
6204	AssertPtr(pVCpu);
6205	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6206	NOREF(pVM);
6207	NOREF(pCtx);
6208
6209	/** @todo this will change with preemption hooks where we only VMCLEAR when
6210	* we are actually going to be preempted, not all the time like we
6211	* currently do. */
6212	/*
6213	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6214	* and mark the VMCS launch-state as "clear".
6215	*/
6216	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6217	return rc;
6218	}
6219
6220
6221	/**
6222	* Saves the host state in the VMCS host-state.
6223	* Sets up the VM-exit MSR-load area.
6224	*
6225	* The CPU state will be loaded from these fields on every successful VM-exit.
6226	*
6227	* @returns VBox status code.
6228	* @param pVM Pointer to the VM.
6229	* @param pVCpu Pointer to the VMCPU.
6230	*
6231	* @remarks No-long-jump zone!!!
6232	*/
6233	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6234	{
6235	AssertPtr(pVM);
6236	AssertPtr(pVCpu);
6237	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6238
6239	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6240
6241	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6242	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6243	return VINF_SUCCESS;
6244
6245	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6246	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6247
6248	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6249	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6250
6251	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6252	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6253
6254	/* Reset the host-state-changed flag. */
6255	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6256	return rc;
6257	}
6258
6259
6260	/**
6261	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6262	* loaded from these fields on every successful VM-entry.
6263	*
6264	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6265	* Sets up the VM-entry controls.
6266	* Sets up the appropriate VMX non-root function to execute guest code based on
6267	* the guest CPU mode.
6268	*
6269	* @returns VBox status code.
6270	* @param pVM Pointer to the VM.
6271	* @param pVCpu Pointer to the VMCPU.
6272	* @param pCtx Pointer to the guest-CPU context.
6273	*
6274	* @remarks No-long-jump zone!!!
6275	*/
6276	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6277	{
6278	AssertPtr(pVM);
6279	AssertPtr(pVCpu);
6280	AssertPtr(pCtx);
6281	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6282
6283	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6284
6285	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6286
6287	/* Determine real-on-v86 mode. */
6288	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6289	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6290	&& CPUMIsGuestInRealModeEx(pCtx))
6291	{
6292	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6293	}
6294
6295	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6296	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6297
6298	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6299	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6300
6301	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6302	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6303
6304	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6305	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6306
6307	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6308	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6309
6310	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6312
6313	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6315
6316	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6318
6319	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6321
6322	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6324
6325	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6326	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6327	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6328
6329	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6330	return rc;
6331	}
6332
6333
6334	/**
6335	* Does the preparations before executing guest code in VT-x.
6336	*
6337	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6338	* recompiler. We must be cautious what we do here regarding committing
6339	* guest-state information into the the VMCS assuming we assuredly execute the
6340	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6341	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6342	* that the recompiler can (and should) use them when it resumes guest
6343	* execution. Otherwise such operations must be done when we can no longer
6344	* exit to ring-3.
6345	*
6346	* @returns VBox status code (informational status codes included).
6347	* @retval VINF_SUCCESS if we can proceed with running the guest.
6348	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6349	* into the guest.
6350	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6351	*
6352	* @param pVM Pointer to the VM.
6353	* @param pVCpu Pointer to the VMCPU.
6354	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6355	* out-of-sync. Make sure to update the required fields
6356	* before using them.
6357	* @param pVmxTransient Pointer to the VMX transient structure.
6358	*
6359	* @remarks Called with preemption disabled.
6360	*/
6361	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6362	{
6363	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6364
6365	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6366	PGMRZDynMapFlushAutoSet(pVCpu);
6367	#endif
6368
6369	/* Check force flag actions that might require us to go back to ring-3. */
6370	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6371	if (rc != VINF_SUCCESS)
6372	return rc;
6373
6374	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6375	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6376	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6377	{
6378	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6379	RTGCPHYS GCPhysApicBase;
6380	GCPhysApicBase = pMixedCtx->msrApicBase;
6381	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6382
6383	/* Unalias any existing mapping. */
6384	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6385	AssertRCReturn(rc, rc);
6386
6387	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6388	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6389	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6390	AssertRCReturn(rc, rc);
6391
6392	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6393	}
6394
6395	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6396	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6397	pVmxTransient->uEFlags = ASMIntDisableFlags();
6398	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6399	{
6400	ASMSetFlags(pVmxTransient->uEFlags);
6401	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6402	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6403	return VINF_EM_RAW_INTERRUPT;
6404	}
6405	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6406	#endif
6407
6408	/*
6409	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6410	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6411	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6412	*/
6413	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6414	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6415	AssertRCReturn(rc, rc);
6416	return rc;
6417	}
6418
6419
6420	/**
6421	* Prepares to run guest code in VT-x and we've committed to doing so. This
6422	* means there is no backing out to ring-3 or anywhere else at this
6423	* point.
6424	*
6425	* @param pVM Pointer to the VM.
6426	* @param pVCpu Pointer to the VMCPU.
6427	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6428	* out-of-sync. Make sure to update the required fields
6429	* before using them.
6430	* @param pVmxTransient Pointer to the VMX transient structure.
6431	*
6432	* @remarks Called with preemption disabled.
6433	* @remarks No-long-jump zone!!!
6434	*/
6435	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6436	{
6437	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6438	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6439
6440	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6441	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6442	pVmxTransient->uEFlags = ASMIntDisableFlags();
6443	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6444	#endif
6445
6446	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6447	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6448	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6449	int rc = VINF_SUCCESS;
6450	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6451	{
6452	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6453	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6454	}
6455	else if (pVCpu->hm.s.fContextUseFlags)
6456	{
6457	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6458	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6459	}
6460	AssertRC(rc);
6461	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6462
6463	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6464	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6465	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6466
6467	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6468	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6469	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6470
6471	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6472	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6473	{
6474	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6475	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6476	}
6477
6478	/*
6479	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6480	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6481	*/
6482	if (pVM->hm.s.fTPRPatchingActive)
6483	{
6484	Assert(!CPUMIsGuestInLongMode(pVCpu));
6485
6486	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6487	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6488	AssertRC(rc);
6489
6490	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6491	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6492	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6493	see hmR0VmxLoadGuestApicState(). */
6494	}
6495
6496	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6497	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6498	to start executing. */
6499	}
6500
6501
6502	/**
6503	* Performs some essential restoration of state after running guest code in
6504	* VT-x.
6505	*
6506	* @param pVM Pointer to the VM.
6507	* @param pVCpu Pointer to the VMCPU.
6508	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6509	* out-of-sync. Make sure to update the required fields
6510	* before using them.
6511	* @param pVmxTransient Pointer to the VMX transient structure.
6512	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6513	*
6514	* @remarks Called with interrupts disabled.
6515	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6516	* unconditionally when it is safe to do so.
6517	*/
6518	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6519	{
6520	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6521	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6522
6523	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6524	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6525	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6526	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6527	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6528
6529	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6530	{
6531	/** @todo Find a way to fix hardcoding a guestimate. */
6532	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6533	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6534	}
6535
6536	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6537	Assert(!(ASMGetFlags() & X86_EFL_IF));
6538	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6539
6540	/* Restore the effects of TPR patching if any. */
6541	if (pVM->hm.s.fTPRPatchingActive)
6542	{
6543	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6544	AssertRC(rc);
6545	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6546	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6547	}
6548
6549	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6550	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6551
6552	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6553	uint32_t uExitReason;
6554	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6555	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6556	AssertRC(rc);
6557	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6558	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6559
6560	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6561	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6562
6563	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6564	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6565	{
6566	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6567	return;
6568	}
6569
6570	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6571	{
6572	/* Update the guest interruptibility-state from the VMCS. */
6573	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6574
6575	/*
6576	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6577	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6578	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6579	*/
6580	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6581	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6582	{
6583	Log(("Setting TPR=%d\n", pVCpu->hm.s.vmx.pbVirtApic[0x80]));
6584	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6585	AssertRC(rc);
6586	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6587	}
6588	}
6589	}
6590
6591
6592	/**
6593	* Runs the guest code using VT-x.
6594	*
6595	* @returns VBox status code.
6596	* @param pVM Pointer to the VM.
6597	* @param pVCpu Pointer to the VMCPU.
6598	* @param pCtx Pointer to the guest-CPU context.
6599	*
6600	* @remarks Called with preemption disabled.
6601	*/
6602	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6603	{
6604	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6605	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6606
6607	VMXTRANSIENT VmxTransient;
6608	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6609	int rc = VERR_INTERNAL_ERROR_5;
6610	unsigned cLoops = 0;
6611
6612	for (;; cLoops++)
6613	{
6614	Assert(!HMR0SuspendPending());
6615	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6616	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6617	(unsigned)RTMpCpuId(), cLoops));
6618
6619	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6620	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6621	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6622	if (rc != VINF_SUCCESS)
6623	break;
6624
6625	/*
6626	* No longjmps to ring-3 from this point on!!!
6627	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6628	* This also disables flushing of the R0-logger instance (if any).
6629	*/
6630	VMMRZCallRing3Disable(pVCpu);
6631	VMMRZCallRing3RemoveNotification(pVCpu);
6632	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6633
6634	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6635	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6636
6637	/*
6638	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6639	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6640	*/
6641	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6642	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6643	{
6644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6645	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6646	return rc;
6647	}
6648
6649	/* Handle the VM-exit. */
6650	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6651	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6652	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6653	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6654	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6655	if (rc != VINF_SUCCESS)
6656	break;
6657	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6658	{
6659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6660	rc = VINF_EM_RAW_INTERRUPT;
6661	break;
6662	}
6663	}
6664
6665	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6666	if (rc == VERR_EM_INTERPRETER)
6667	rc = VINF_EM_RAW_EMULATE_INSTR;
6668	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6669	return rc;
6670	}
6671
6672	#if 0
6673	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, unsigned rcReason)
6674	{
6675	int rc;
6676	switch (rcReason)
6677	{
6678	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6679	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6680	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6681	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6682	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6683	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6684	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6691	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6692	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6693	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6695	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6697	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6699	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6700	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6701	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6702	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6703	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6704	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6705	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6706	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6707	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6708	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6709	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6710	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6711
6712	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6713	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6714	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6715	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6716	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6717	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6718	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6719	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6720	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6721
6722	case VMX_EXIT_VMCALL:
6723	case VMX_EXIT_VMCLEAR:
6724	case VMX_EXIT_VMLAUNCH:
6725	case VMX_EXIT_VMPTRLD:
6726	case VMX_EXIT_VMPTRST:
6727	case VMX_EXIT_VMREAD:
6728	case VMX_EXIT_VMRESUME:
6729	case VMX_EXIT_VMWRITE:
6730	case VMX_EXIT_VMXOFF:
6731	case VMX_EXIT_VMXON:
6732	case VMX_EXIT_INVEPT:
6733	case VMX_EXIT_INVVPID:
6734	case VMX_EXIT_VMFUNC:
6735	rc = hmR0VmxExitInjectXcptUD(pVM, pVCpu, pMixedCtx, pVmxTransient);
6736	break;
6737	default:
6738	rc = hmR0VmxExitErrUndefined(pVM, pVCpu, pMixedCtx, pVmxTransient);
6739	break;
6740	}
6741	return rc;
6742	}
6743	#endif
6744
6745	#ifdef DEBUG
6746	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6747	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6748	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6749	# define VMX_ASSERT_PREEMPT_CPUID() \
6750	do \
6751	{ \
6752	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6753	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6754	} while (0)
6755
6756	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6757	do { \
6758	AssertPtr(pVM); \
6759	AssertPtr(pVCpu); \
6760	AssertPtr(pMixedCtx); \
6761	AssertPtr(pVmxTransient); \
6762	Assert(pVmxTransient->fVMEntryFailed == false); \
6763	Assert(ASMIntAreEnabled()); \
6764	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6765	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6766	LogFunc(("\n")); \
6767	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6768	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6769	VMX_ASSERT_PREEMPT_CPUID(); \
6770	} while (0)
6771	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6772	do { \
6773	LogFunc(("\n")); \
6774	} while(0)
6775	#else /* Release builds */
6776	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6777	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6778	#endif
6779
6780
6781	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6782	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6783	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6784	/**
6785	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6786	*/
6787	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6788	{
6789	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6791	return VINF_SUCCESS;
6792	}
6793
6794
6795	/**
6796	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6797	*/
6798	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6799	{
6800	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6801	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6802	AssertRCReturn(rc, rc);
6803
6804	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6805	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6806	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6807
6808	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6809	return VINF_EM_RAW_INTERRUPT;
6810
6811	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6812	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6813	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6814	return VINF_SUCCESS;
6815	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6816	return rc;
6817
6818	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6819	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6820	switch (uIntrType)
6821	{
6822	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6823	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6824	/* no break */
6825	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6826	{
6827	switch (uVector)
6828	{
6829	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6830	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6831	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6832	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6833	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6834	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6835	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6836	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6837	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6838	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6839	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6840	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6841	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6842	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6843	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6844	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6845	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6846	#endif
6847	default:
6848	{
6849	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6850	AssertRCReturn(rc, rc);
6851
6852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6853	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6854	{
6855	Assert(pVM->hm.s.vmx.pRealModeTSS);
6856	Assert(PDMVmmDevHeapIsEnabled(pVM));
6857	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6858	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6859	AssertRCReturn(rc, rc);
6860	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6861	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6862	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6863	AssertRCReturn(rc, rc);
6864	}
6865	else
6866	{
6867	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6868	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6869	}
6870	break;
6871	}
6872	}
6873	break;
6874	}
6875
6876	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6877	default:
6878	{
6879	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6880	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6881	break;
6882	}
6883	}
6884	return rc;
6885	}
6886
6887
6888	/**
6889	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6890	*/
6891	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6892	{
6893	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6894
6895	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6896	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
6897	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6898	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6899	AssertRCReturn(rc, rc);
6900
6901	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6903	return VINF_SUCCESS;
6904	}
6905
6906
6907	/**
6908	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6909	*/
6910	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6911	{
6912	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6913	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6914	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6915	}
6916
6917
6918	/**
6919	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6920	*/
6921	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6922	{
6923	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6924	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6925	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6926	AssertRCReturn(rc, rc);
6927
6928	pMixedCtx->rip += pVmxTransient->cbInstr;
6929	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6930
6931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6932	return VINF_SUCCESS;
6933	}
6934
6935
6936	/**
6937	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
6938	*/
6939	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6940	{
6941	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6942	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6943	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6944	AssertRCReturn(rc, rc);
6945
6946	pMixedCtx->rip += pVmxTransient->cbInstr;
6947	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6948
6949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6950	return VINF_SUCCESS;
6951	}
6952
6953
6954	/**
6955	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6956	*/
6957	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6958	{
6959	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6960	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6961	if (RT_LIKELY(rc == VINF_SUCCESS))
6962	{
6963	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6964	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6965	AssertRCReturn(rc, rc);
6966	Assert(pVmxTransient->cbInstr == 2);
6967
6968	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6969	pMixedCtx->rip += pVmxTransient->cbInstr;
6970	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6971	}
6972	else
6973	{
6974	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6975	rc = VERR_EM_INTERPRETER;
6976	}
6977	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6978	return rc;
6979	}
6980
6981
6982	/**
6983	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6984	*/
6985	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6986	{
6987	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6988	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6989	AssertRCReturn(rc, rc);
6990
6991	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6992	return VINF_EM_RAW_EMULATE_INSTR;
6993
6994	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6995	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6996	}
6997
6998
6999	/**
7000	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7001	*/
7002	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7003	{
7004	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7005	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7006	AssertRCReturn(rc, rc);
7007
7008	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7009	if (RT_LIKELY(rc == VINF_SUCCESS))
7010	{
7011	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7012	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7013	AssertRCReturn(rc, rc);
7014	Assert(pVmxTransient->cbInstr == 2);
7015
7016	pMixedCtx->rip += pVmxTransient->cbInstr;
7017	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7018
7019	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7020	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7021	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7022	}
7023	else
7024	{
7025	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7026	rc = VERR_EM_INTERPRETER;
7027	}
7028	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7029	return rc;
7030	}
7031
7032
7033	/**
7034	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7035	*/
7036	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7037	{
7038	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7039	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7040	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7041	AssertRCReturn(rc, rc);
7042
7043	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7044	if (RT_LIKELY(rc == VINF_SUCCESS))
7045	{
7046	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7047	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7048	AssertRCReturn(rc, rc);
7049	Assert(pVmxTransient->cbInstr == 3);
7050
7051	pMixedCtx->rip += pVmxTransient->cbInstr;
7052	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7053
7054	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7055	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7056	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7057	}
7058	else
7059	{
7060	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7061	rc = VERR_EM_INTERPRETER;
7062	}
7063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7064	return rc;
7065	}
7066
7067
7068	/**
7069	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7070	*/
7071	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7072	{
7073	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7074	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7075	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7076	AssertRCReturn(rc, rc);
7077
7078	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7079	if (RT_LIKELY(rc == VINF_SUCCESS))
7080	{
7081	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7082	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7083	AssertRCReturn(rc, rc);
7084	Assert(pVmxTransient->cbInstr == 2);
7085
7086	pMixedCtx->rip += pVmxTransient->cbInstr;
7087	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7088	}
7089	else
7090	{
7091	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7092	rc = VERR_EM_INTERPRETER;
7093	}
7094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7095	return rc;
7096	}
7097
7098
7099	/**
7100	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7101	*/
7102	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7103	{
7104	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7105	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7106	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7107	AssertRCReturn(rc, rc);
7108
7109	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7110	rc = VBOXSTRICTRC_VAL(rc2);
7111	if (RT_LIKELY(rc == VINF_SUCCESS))
7112	{
7113	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7114	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7115	AssertRCReturn(rc, rc);
7116
7117	pMixedCtx->rip += pVmxTransient->cbInstr;
7118	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7119	}
7120	else
7121	{
7122	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7123	pVmxTransient->uExitQualification, rc));
7124	rc = VERR_EM_INTERPRETER;
7125	}
7126	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7127	return rc;
7128	}
7129
7130
7131	/**
7132	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7133	*/
7134	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7135	{
7136	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7137	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7138	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7139	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7140	AssertRCReturn(rc, rc);
7141
7142	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7143	if (RT_LIKELY(rc == VINF_SUCCESS))
7144	{
7145	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7146	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7147	AssertRCReturn(rc, rc);
7148
7149	pMixedCtx->rip += pVmxTransient->cbInstr;
7150	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7151	}
7152	else
7153	{
7154	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7155	rc = VERR_EM_INTERPRETER;
7156	}
7157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7158	return rc;
7159	}
7160
7161
7162	/**
7163	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7164	*/
7165	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7166	{
7167	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7168	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7169	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7170	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7171	AssertRCReturn(rc, rc);
7172
7173	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7174	rc = VBOXSTRICTRC_VAL(rc2);
7175	if (RT_LIKELY( rc == VINF_SUCCESS
7176	\|\| rc == VINF_EM_HALT))
7177	{
7178	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7179	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7180	AssertRCReturn(rc3, rc3);
7181
7182	pMixedCtx->rip += pVmxTransient->cbInstr;
7183	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7184
7185	if ( rc == VINF_EM_HALT
7186	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7187	{
7188	rc = VINF_SUCCESS;
7189	}
7190	}
7191	else
7192	{
7193	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7194	rc = VERR_EM_INTERPRETER;
7195	}
7196	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7197	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7199	return rc;
7200	}
7201
7202
7203	/**
7204	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7205	*/
7206	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7207	{
7208	/*
7209	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7210	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7211	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7212	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7213	*/
7214	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7215	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7216	}
7217
7218
7219	/**
7220	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7221	*/
7222	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7223	{
7224	/*
7225	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7226	* root operation. If we get there there is something funny going on.
7227	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7228	*/
7229	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7230	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7231	}
7232
7233
7234	/**
7235	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7236	*/
7237	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7238	{
7239	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7240	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7241	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7242	}
7243
7244
7245	/**
7246	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7247	*/
7248	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7249	{
7250	/*
7251	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7252	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7253	* See Intel spec. 25.3 "Other Causes of VM-exits".
7254	*/
7255	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7256	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7257	}
7258
7259
7260	/**
7261	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7262	* VM-exit.
7263	*/
7264	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7265	{
7266	/*
7267	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7268	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7269	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7270	*/
7271	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7272	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7273	}
7274
7275
7276	/**
7277	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7278	* VM-exit.
7279	*/
7280	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7281	{
7282	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7283	return VINF_EM_RESET;
7284	}
7285
7286
7287	/**
7288	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7289	*/
7290	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7291	{
7292	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7293	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7294	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7295	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7296	AssertRCReturn(rc, rc);
7297
7298	pMixedCtx->rip++;
7299	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7300	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7301	rc = VINF_SUCCESS;
7302	else
7303	rc = VINF_EM_HALT;
7304
7305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7306	return rc;
7307	}
7308
7309
7310	/**
7311	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7312	*/
7313	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7314	{
7315	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7316	return hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7317	}
7318
7319
7320	/**
7321	* VM-exit handler for expiry of the VMX preemption timer.
7322	*/
7323	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7324	{
7325	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7326
7327	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7328	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7329	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7330
7331	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7332	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7334	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7335	}
7336
7337
7338	/**
7339	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7340	*/
7341	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342	{
7343	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7344	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7345	/** @todo check if XSETBV is supported by the recompiler. */
7346	return VERR_EM_INTERPRETER;
7347	}
7348
7349
7350	/**
7351	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7352	*/
7353	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7354	{
7355	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7356	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7357	/** @todo implement EMInterpretInvpcid() */
7358	return VERR_EM_INTERPRETER;
7359	}
7360
7361
7362	/**
7363	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7364	* Error VM-exit.
7365	*/
7366	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7367	{
7368	uint32_t uIntrState;
7369	RTHCUINTREG uHCReg;
7370	uint64_t u64Val;
7371
7372	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7373	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7374	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7375	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7376	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7377	AssertRCReturn(rc, rc);
7378
7379	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7380	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7381	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7382	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7383
7384	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7385	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7386	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7387	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7388	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7389	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7390	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7391	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7392	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7393	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7394	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7395	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7396
7397	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7398
7399	return VERR_VMX_INVALID_GUEST_STATE;
7400	}
7401
7402
7403	/**
7404	* VM-exit handler for VM-entry failure due to an MSR-load
7405	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7406	*/
7407	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7408	{
7409	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7410	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7411	}
7412
7413
7414	/**
7415	* VM-exit handler for VM-entry failure due to a machine-check event
7416	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7417	*/
7418	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7419	{
7420	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7421	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7422	}
7423
7424
7425	/**
7426	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7427	* theory.
7428	*/
7429	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7430	{
7431	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7432	pVM, pVCpu, pMixedCtx));
7433	return VERR_VMX_UNDEFINED_EXIT_CODE;
7434	}
7435
7436
7437	/**
7438	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7439	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7440	* Conditional VM-exit.
7441	*/
7442	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7443	{
7444	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7445	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7447	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7448	return VERR_EM_INTERPRETER;
7449	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7450	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7451	}
7452
7453
7454	/**
7455	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7456	*/
7457	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7458	{
7459	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7460	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7462	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7463	return VERR_EM_INTERPRETER;
7464	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7465	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7466	}
7467
7468
7469	/**
7470	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7471	*/
7472	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7473	{
7474	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7475	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7476	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7477	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7478	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7479	AssertRCReturn(rc, rc);
7480
7481	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7482	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7483	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7484	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7485
7486	/* Update RIP and continue guest execution. */
7487	if (RT_LIKELY(rc == VINF_SUCCESS))
7488	{
7489	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7490	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7491	AssertRCReturn(rc, rc);
7492
7493	Assert(pVmxTransient->cbInstr == 2);
7494	pMixedCtx->rip += pVmxTransient->cbInstr;
7495	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7496	}
7497	return rc;
7498	}
7499
7500
7501	/**
7502	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7503	*/
7504	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7505	{
7506	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7507	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7508	AssertRCReturn(rc, rc);
7509	Assert(pVmxTransient->cbInstr == 2);
7510
7511	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7512	if ( pVM->hm.s.fTPRPatchingActive
7513	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7514	{
7515	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7516	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7517	{
7518	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7519	AssertRC(rc);
7520	}
7521
7522	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7523	AssertRCReturn(rc, rc);
7524	pMixedCtx->rip += pVmxTransient->cbInstr;
7525	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7526	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7527	return VINF_SUCCESS;
7528	}
7529
7530	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7531	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7532	{
7533	switch (pMixedCtx->ecx)
7534	{
7535	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7536	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7537	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7538	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7539	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7540	}
7541	}
7542	#ifdef DEBUG
7543	else
7544	{
7545	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7546	switch (pMixedCtx->ecx)
7547	{
7548	case MSR_IA32_SYSENTER_CS:
7549	case MSR_IA32_SYSENTER_EIP:
7550	case MSR_IA32_SYSENTER_ESP:
7551	case MSR_K8_FS_BASE:
7552	case MSR_K8_GS_BASE:
7553	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7554	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7555	case MSR_K8_LSTAR:
7556	case MSR_K6_STAR:
7557	case MSR_K8_SF_MASK:
7558	case MSR_K8_TSC_AUX:
7559	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7560	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7561	}
7562	}
7563	#endif
7564
7565	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7566	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7567	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7568	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7569	AssertRCReturn(rc, rc);
7570
7571	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7572	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7573	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7574
7575	/* Update guest-state and continue execution. */
7576	if (RT_LIKELY(rc == VINF_SUCCESS))
7577	{
7578	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7579	AssertRCReturn(rc, rc);
7580
7581	pMixedCtx->rip += pVmxTransient->cbInstr;
7582	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7583
7584	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7585	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7586	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7587	{
7588	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7589	}
7590	}
7591	return rc;
7592	}
7593
7594
7595	/**
7596	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7597	*/
7598	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7599	{
7600	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7601	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7603	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7604	return VERR_EM_INTERPRETER;
7605	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7606	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7607	}
7608
7609
7610	/**
7611	* VM-exit handler for when the TPR value is lowered below the specified
7612	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7613	*/
7614	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7615	{
7616	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7617	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7618
7619	/*
7620	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7621	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7622	* resume guest execution.
7623	*/
7624	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7625	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7626	return VINF_SUCCESS;
7627	}
7628
7629
7630	/**
7631	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7632	* VM-exit.
7633	*
7634	* @retval VINF_SUCCESS when guest execution can continue.
7635	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7636	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7637	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7638	* recompiler.
7639	*/
7640	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7641	{
7642	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7643	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7644	AssertRCReturn(rc, rc);
7645
7646	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7647	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7648	switch (uAccessType)
7649	{
7650	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7651	{
7652	#if 0
7653	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7654	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7655	#else
7656	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7657	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7658	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7659	#endif
7660	AssertRCReturn(rc, rc);
7661
7662	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7663	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7664	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7665	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7666
7667	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7668	{
7669	case 0: /* CR0 */
7670	Log(("CR0 write rc=%d\n", rc));
7671	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7672	break;
7673	case 2: /* CR2 */
7674	Log(("CR2 write rc=%d\n", rc));
7675	break;
7676	case 3: /* CR3 */
7677	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7678	Log(("CR3 write rc=%d\n", rc));
7679	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7680	break;
7681	case 4: /* CR4 */
7682	Log(("CR4 write rc=%d\n", rc));
7683	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7684	break;
7685	case 8: /* CR8 */
7686	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7687	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7688	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7689	break;
7690	default:
7691	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7692	break;
7693	}
7694
7695	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7696	break;
7697	}
7698
7699	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7700	{
7701	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7702	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7703	AssertRCReturn(rc, rc);
7704	Assert( !pVM->hm.s.fNestedPaging
7705	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7706	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7707
7708	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7709	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7710	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7711
7712	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7713	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7714	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7715	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7717	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7718	break;
7719	}
7720
7721	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7722	{
7723	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7724	AssertRCReturn(rc, rc);
7725	rc = EMInterpretCLTS(pVM, pVCpu);
7726	AssertRCReturn(rc, rc);
7727	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7728	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7729	Log(("CRX CLTS write rc=%d\n", rc));
7730	break;
7731	}
7732
7733	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7734	{
7735	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7736	AssertRCReturn(rc, rc);
7737	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7738	if (RT_LIKELY(rc == VINF_SUCCESS))
7739	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7741	Log(("CRX LMSW write rc=%d\n", rc));
7742	break;
7743	}
7744
7745	default:
7746	{
7747	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7748	return VERR_VMX_UNEXPECTED_EXCEPTION;
7749	}
7750	}
7751
7752	/* Validate possible error codes. */
7753	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7754	if (RT_SUCCESS(rc))
7755	{
7756	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7757	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7758	AssertRCReturn(rc2, rc2);
7759	pMixedCtx->rip += pVmxTransient->cbInstr;
7760	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7761	}
7762
7763	return rc;
7764	}
7765
7766
7767	/**
7768	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7769	* VM-exit.
7770	*/
7771	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7772	{
7773	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7774
7775	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7776	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7777	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7778	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7779	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7780	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7781	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7782	AssertRCReturn(rc, rc);
7783
7784	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7785	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7786	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7787	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7788	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7789	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7790	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7791
7792	/* I/O operation lookup arrays. */
7793	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7794	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7795
7796	const uint32_t cbSize = s_aIOSize[uIOWidth];
7797	const uint32_t cbInstr = pVmxTransient->cbInstr;
7798	if (fIOString)
7799	{
7800	/* INS/OUTS - I/O String instruction. */
7801	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7802	/** @todo for now manually disassemble later optimize by getting the fields from
7803	* the VMCS. */
7804	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7805	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7806	* segment prefix info. */
7807	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7808	if (RT_SUCCESS(rc))
7809	{
7810	if (fIOWrite)
7811	{
7812	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7813	(DISCPUMODE)pDis->uAddrMode, cbSize);
7814	rc = VBOXSTRICTRC_VAL(rc2);
7815	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7816	}
7817	else
7818	{
7819	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7820	(DISCPUMODE)pDis->uAddrMode, cbSize);
7821	rc = VBOXSTRICTRC_VAL(rc2);
7822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7823	}
7824	}
7825	else
7826	{
7827	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7828	rc = VINF_EM_RAW_EMULATE_INSTR;
7829	}
7830	}
7831	else
7832	{
7833	/* IN/OUT - I/O instruction. */
7834	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7835	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7836	if (fIOWrite)
7837	{
7838	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7839	rc = VBOXSTRICTRC_VAL(rc2);
7840	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7841	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7843	}
7844	else
7845	{
7846	uint32_t u32Result = 0;
7847	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7848	rc = VBOXSTRICTRC_VAL(rc2);
7849	if (IOM_SUCCESS(rc))
7850	{
7851	/* Save result of I/O IN instr. in AL/AX/EAX. */
7852	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7853	}
7854	else if (rc == VINF_IOM_R3_IOPORT_READ)
7855	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7857	}
7858	}
7859
7860	if (IOM_SUCCESS(rc))
7861	{
7862	pMixedCtx->rip += cbInstr;
7863	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7864	if (RT_LIKELY(rc == VINF_SUCCESS))
7865	{
7866	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7867	AssertRCReturn(rc, rc);
7868
7869	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7870	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7871	{
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7873	for (unsigned i = 0; i < 4; i++)
7874	{
7875	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7876	if ( ( uIOPort >= pMixedCtx->dr[i]
7877	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7878	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7879	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7880	{
7881	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7882	uint64_t uDR6 = ASMGetDR6();
7883
7884	/* Clear all breakpoint status flags and set the one we just hit. */
7885	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7886	uDR6 \|= (uint64_t)RT_BIT(i);
7887
7888	/*
7889	* Note: AMD64 Architecture Programmer's Manual 13.1:
7890	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7891	* be cleared by software after the contents have been read.
7892	*/
7893	ASMSetDR6(uDR6);
7894
7895	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7896	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7897
7898	/* Paranoia. */
7899	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7900	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7901	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7902
7903	/* Resync DR7 */
7904	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7905	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7906
7907	/* Inject #DB and get on with guest execution. */
7908	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7909	AssertRCReturn(rc, rc);
7910	break;
7911	}
7912	}
7913	}
7914	}
7915	}
7916
7917	#ifdef DEBUG
7918	if (rc == VINF_IOM_R3_IOPORT_READ)
7919	Assert(!fIOWrite);
7920	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7921	Assert(fIOWrite);
7922	else
7923	{
7924	AssertMsg( RT_FAILURE(rc)
7925	\|\| rc == VINF_SUCCESS
7926	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7927	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7928	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7929	}
7930	#endif
7931
7932	return rc;
7933	}
7934
7935
7936	/**
7937	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7938	* VM-exit.
7939	*/
7940	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7941	{
7942	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7943
7944	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7945	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7946	AssertRCReturn(rc, rc);
7947	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7948	{
7949	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7950	AssertRCReturn(rc, rc);
7951	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7952	{
7953	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7954	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7955	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7956	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7957	{
7958	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7959	pVCpu->hm.s.Event.fPending = true;
7960	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7961	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7962	AssertRCReturn(rc, rc);
7963	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7964	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7965	else
7966	pVCpu->hm.s.Event.u32ErrCode = 0;
7967	}
7968	}
7969	}
7970	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7971	* emulation. */
7972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7973	return VERR_EM_INTERPRETER;
7974	}
7975
7976
7977	/**
7978	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7979	*/
7980	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7981	{
7982	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7983	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7984	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7985	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7986	AssertRCReturn(rc, rc);
7987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7988	return VINF_EM_DBG_STOP;
7989	}
7990
7991
7992	/**
7993	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7994	*/
7995	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7996	{
7997	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7998	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7999
8000	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8001	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8002	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8003	return VINF_SUCCESS;
8004	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8005	return rc;
8006
8007	#if 0
8008	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8009	* just sync the whole thing. */
8010	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8011	#else
8012	/* Aggressive state sync. for now. */
8013	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8014	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8015	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8016	#endif
8017	AssertRCReturn(rc, rc);
8018
8019	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8020	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8021	switch (uAccessType)
8022	{
8023	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8024	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8025	{
8026	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8027	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8028	{
8029	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8030	}
8031
8032	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8033	GCPhys &= PAGE_BASE_GC_MASK;
8034	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8035	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8036	CPUMCTX2CORE(pMixedCtx), GCPhys);
8037	rc = VBOXSTRICTRC_VAL(rc2);
8038	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8039	if ( rc == VINF_SUCCESS
8040	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8041	\|\| rc == VERR_PAGE_NOT_PRESENT)
8042	{
8043	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8044	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8045	rc = VINF_SUCCESS;
8046	}
8047	break;
8048	}
8049
8050	default:
8051	rc = VINF_EM_RAW_EMULATE_INSTR;
8052	break;
8053	}
8054
8055	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8056	return rc;
8057	}
8058
8059
8060	/**
8061	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8062	* VM-exit.
8063	*/
8064	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8065	{
8066	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8067
8068	/* We should -not- get this VM-exit if the guest is debugging. */
8069	if (CPUMIsGuestDebugStateActive(pVCpu))
8070	{
8071	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
8072	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8073	}
8074
8075	int rc = VERR_INTERNAL_ERROR_5;
8076	if ( !DBGFIsStepping(pVCpu)
8077	&& !CPUMIsHyperDebugStateActive(pVCpu))
8078	{
8079	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8080
8081	/* Don't intercept MOV DRx. */
8082	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8083	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8084	AssertRCReturn(rc, rc);
8085
8086	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8087	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8088	AssertRC(rc);
8089	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8090
8091	#ifdef VBOX_WITH_STATISTICS
8092	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8093	AssertRCReturn(rc, rc);
8094	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8096	else
8097	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8098	#endif
8099	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8100	return VINF_SUCCESS;
8101	}
8102
8103	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8104	* time and restore DRx registers afterwards */
8105	/*
8106	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8107	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8108	*/
8109	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8110	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8111	AssertRCReturn(rc, rc);
8112
8113	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8114	{
8115	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8116	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8117	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8118	if (RT_SUCCESS(rc))
8119	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8120	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8121	}
8122	else
8123	{
8124	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8125	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8126	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8128	}
8129
8130	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8131	if (RT_SUCCESS(rc))
8132	{
8133	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8134	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8135	AssertRCReturn(rc2, rc2);
8136	pMixedCtx->rip += pVmxTransient->cbInstr;
8137	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8138	}
8139	return rc;
8140	}
8141
8142
8143	/**
8144	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8145	* Conditional VM-exit.
8146	*/
8147	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8148	{
8149	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8150	Assert(pVM->hm.s.fNestedPaging);
8151
8152	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8153	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8154	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8155	return VINF_SUCCESS;
8156	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8157	return rc;
8158
8159	RTGCPHYS GCPhys = 0;
8160	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8161
8162	#if 0
8163	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8164	#else
8165	/* Aggressive state sync. for now. */
8166	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8167	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8168	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8169	#endif
8170	AssertRCReturn(rc, rc);
8171
8172	/*
8173	* If we succeed, resume guest execution.
8174	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8175	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8176	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8177	* weird case. See @bugref{6043}.
8178	*/
8179	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8180	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8181	rc = VBOXSTRICTRC_VAL(rc2);
8182	if ( rc == VINF_SUCCESS
8183	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8184	\|\| rc == VERR_PAGE_NOT_PRESENT)
8185	{
8186	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8187	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8188	return VINF_SUCCESS;
8189	}
8190	return rc;
8191	}
8192
8193
8194	/**
8195	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8196	* VM-exit.
8197	*/
8198	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8199	{
8200	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8201	Assert(pVM->hm.s.fNestedPaging);
8202
8203	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8204	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8205	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8206	return VINF_SUCCESS;
8207	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8208	return rc;
8209
8210	RTGCPHYS GCPhys = 0;
8211	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8212	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8213	#if 0
8214	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8215	#else
8216	/* Aggressive state sync. for now. */
8217	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8218	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8219	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8220	#endif
8221	AssertRCReturn(rc, rc);
8222
8223	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8224	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8225
8226	RTGCUINT uErrorCode = 0;
8227	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8228	uErrorCode \|= X86_TRAP_PF_ID;
8229	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8230	uErrorCode \|= X86_TRAP_PF_RW;
8231	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8232	uErrorCode \|= X86_TRAP_PF_P;
8233
8234	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8235	TRPMSetErrorCode(pVCpu, uErrorCode);
8236	TRPMSetFaultAddress(pVCpu, GCPhys);
8237
8238	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8239	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8240
8241	/* Handle the pagefault trap for the nested shadow table. */
8242	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8243	TRPMResetTrap(pVCpu);
8244
8245	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8246	if ( rc == VINF_SUCCESS
8247	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8248	\|\| rc == VERR_PAGE_NOT_PRESENT)
8249	{
8250	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8251	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8252	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8253	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8254	return VINF_SUCCESS;
8255	}
8256
8257	Log(("EPT return to ring-3 rc=%d\n"));
8258	return rc;
8259	}
8260
8261
8262	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8263	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8264	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8265	/**
8266	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8267	*/
8268	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8269	{
8270	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8271
8272	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8273	AssertRCReturn(rc, rc);
8274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8275
8276	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8277	{
8278	/* Old-style FPU error reporting needs some extra work. */
8279	/** @todo don't fall back to the recompiler, but do it manually. */
8280	return VERR_EM_INTERPRETER;
8281	}
8282	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8283	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8284	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8285	AssertRCReturn(rc, rc);
8286	return rc;
8287	}
8288
8289
8290	/**
8291	* VM-exit exception handler for #BP (Breakpoint exception).
8292	*/
8293	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8294	{
8295	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8296
8297	/** @todo Try optimize this by not saving the entire guest state unless
8298	* really needed. */
8299	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8300	AssertRCReturn(rc, rc);
8301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8302
8303	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8304	if (rc == VINF_EM_RAW_GUEST_TRAP)
8305	{
8306	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8307	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8308	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8309	AssertRCReturn(rc, rc);
8310
8311	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8312	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8313	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8314	AssertRCReturn(rc, rc);
8315	}
8316
8317	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8318	return rc;
8319	}
8320
8321
8322	/**
8323	* VM-exit exception handler for #DB (Debug exception).
8324	*/
8325	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8326	{
8327	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8328
8329	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8330	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8331	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8332	AssertRCReturn(rc, rc);
8333
8334	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8335	uint64_t uDR6 = X86_DR6_INIT_VAL;
8336	uDR6 \|= (pVmxTransient->uExitQualification
8337	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8338	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8340	if (rc == VINF_EM_RAW_GUEST_TRAP)
8341	{
8342	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8343	pMixedCtx->dr[6] = uDR6;
8344
8345	if (CPUMIsGuestDebugStateActive(pVCpu))
8346	ASMSetDR6(pMixedCtx->dr[6]);
8347
8348	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8349	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8350
8351	/* Paranoia. */
8352	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8353	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8354	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8355
8356	/* Resync DR7. */
8357	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8358
8359	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8360	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8361	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8362	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8363	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8364	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8365	AssertRCReturn(rc,rc);
8366	return rc;
8367	}
8368	/* Return to ring 3 to deal with the debug exit code. */
8369	return rc;
8370	}
8371
8372
8373	/**
8374	* VM-exit exception handler for #NM (Device-not-available exception: floating
8375	* point exception).
8376	*/
8377	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8378	{
8379	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8380
8381	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8382	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8383	#endif
8384
8385	/* We require CR0 and EFER. EFER is always up-to-date. */
8386	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8387	AssertRCReturn(rc, rc);
8388
8389	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8390	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8391	if (rc == VINF_SUCCESS)
8392	{
8393	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8394	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8395	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8396	return VINF_SUCCESS;
8397	}
8398
8399	/* Forward #NM to the guest. */
8400	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8401	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8402	AssertRCReturn(rc, rc);
8403	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8404	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8405	pVmxTransient->cbInstr, 0 /* error code */);
8406	AssertRCReturn(rc, rc);
8407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8408	return rc;
8409	}
8410
8411
8412	/**
8413	* VM-exit exception handler for #GP (General-protection exception).
8414	*
8415	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8416	*/
8417	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8418	{
8419	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8420
8421	int rc = VERR_INTERNAL_ERROR_5;
8422
8423	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8424	{
8425	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8426	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8427	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8428	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8429	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8430	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8431	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8432	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8433	AssertRCReturn(rc, rc);
8434	return rc;
8435	#else
8436	/* We don't intercept #GP. */
8437	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8438	return VERR_VMX_UNEXPECTED_EXCEPTION;
8439	#endif
8440	}
8441
8442	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8443	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8444
8445	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8446	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8447	AssertRCReturn(rc, rc);
8448
8449	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8450	unsigned int cbOp = 0;
8451	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8452	if (RT_SUCCESS(rc))
8453	{
8454	rc = VINF_SUCCESS;
8455	Assert(cbOp == pDis->cbInstr);
8456	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8457	switch (pDis->pCurInstr->uOpcode)
8458	{
8459	case OP_CLI:
8460	pMixedCtx->eflags.Bits.u1IF = 0;
8461	pMixedCtx->rip += pDis->cbInstr;
8462	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8463	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8464	break;
8465
8466	case OP_STI:
8467	pMixedCtx->eflags.Bits.u1IF = 1;
8468	pMixedCtx->rip += pDis->cbInstr;
8469	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8470	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8471	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8473	break;
8474
8475	case OP_HLT:
8476	rc = VINF_EM_HALT;
8477	pMixedCtx->rip += pDis->cbInstr;
8478	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8480	break;
8481
8482	case OP_POPF:
8483	{
8484	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8485	uint32_t cbParm = 0;
8486	uint32_t uMask = 0;
8487	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8488	{
8489	cbParm = 4;
8490	uMask = 0xffffffff;
8491	}
8492	else
8493	{
8494	cbParm = 2;
8495	uMask = 0xffff;
8496	}
8497
8498	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8499	RTGCPTR GCPtrStack = 0;
8500	X86EFLAGS uEflags;
8501	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8502	&GCPtrStack);
8503	if (RT_SUCCESS(rc))
8504	{
8505	Assert(sizeof(uEflags.u32) >= cbParm);
8506	uEflags.u32 = 0;
8507	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8508	}
8509	if (RT_FAILURE(rc))
8510	{
8511	rc = VERR_EM_INTERPRETER;
8512	break;
8513	}
8514	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8515	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8516	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8517	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8518	pMixedCtx->eflags.Bits.u1RF = 0;
8519	pMixedCtx->esp += cbParm;
8520	pMixedCtx->esp &= uMask;
8521	pMixedCtx->rip += pDis->cbInstr;
8522	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8524	break;
8525	}
8526
8527	case OP_PUSHF:
8528	{
8529	uint32_t cbParm = 0;
8530	uint32_t uMask = 0;
8531	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8532	{
8533	cbParm = 4;
8534	uMask = 0xffffffff;
8535	}
8536	else
8537	{
8538	cbParm = 2;
8539	uMask = 0xffff;
8540	}
8541
8542	/* Get the stack pointer & push the contents of eflags onto the stack. */
8543	RTGCPTR GCPtrStack = 0;
8544	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8545	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8546	if (RT_FAILURE(rc))
8547	{
8548	rc = VERR_EM_INTERPRETER;
8549	break;
8550	}
8551	X86EFLAGS uEflags;
8552	uEflags = pMixedCtx->eflags;
8553	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8554	uEflags.Bits.u1RF = 0;
8555	uEflags.Bits.u1VM = 0;
8556
8557	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8558	if (RT_FAILURE(rc))
8559	{
8560	rc = VERR_EM_INTERPRETER;
8561	break;
8562	}
8563	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8564	pMixedCtx->esp -= cbParm;
8565	pMixedCtx->esp &= uMask;
8566	pMixedCtx->rip += pDis->cbInstr;
8567	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8569	break;
8570	}
8571
8572	case OP_IRET:
8573	{
8574	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8575	* instruction reference. */
8576	RTGCPTR GCPtrStack = 0;
8577	uint32_t uMask = 0xffff;
8578	uint16_t aIretFrame[3];
8579	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8580	{
8581	rc = VERR_EM_INTERPRETER;
8582	break;
8583	}
8584	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8585	&GCPtrStack);
8586	if (RT_SUCCESS(rc))
8587	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8588	if (RT_FAILURE(rc))
8589	{
8590	rc = VERR_EM_INTERPRETER;
8591	break;
8592	}
8593	pMixedCtx->eip = 0;
8594	pMixedCtx->ip = aIretFrame[0];
8595	pMixedCtx->cs.Sel = aIretFrame[1];
8596	pMixedCtx->cs.ValidSel = aIretFrame[1];
8597	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8598	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8599	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8600	pMixedCtx->sp += sizeof(aIretFrame);
8601	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8602	\| HM_CHANGED_GUEST_RFLAGS;
8603	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8605	break;
8606	}
8607
8608	case OP_INT:
8609	{
8610	uint16_t uVector = pDis->Param1.uValue & 0xff;
8611	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8612	AssertRCReturn(rc, rc);
8613	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8614	break;
8615	}
8616
8617	case OP_INTO:
8618	{
8619	if (pMixedCtx->eflags.Bits.u1OF)
8620	{
8621	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8623	}
8624	break;
8625	}
8626
8627	default:
8628	{
8629	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8630	EMCODETYPE_SUPERVISOR);
8631	rc = VBOXSTRICTRC_VAL(rc2);
8632	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8633	Log2(("#GP rc=%Rrc\n", rc));
8634	break;
8635	}
8636	}
8637	}
8638	else
8639	rc = VERR_EM_INTERPRETER;
8640
8641	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8642	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8643	("#GP Unexpected rc=%Rrc\n", rc));
8644	return rc;
8645	}
8646
8647
8648	/**
8649	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8650	* the exception reported in the VMX transient structure back into the VM.
8651	*
8652	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8653	* VMX transient structure to be up-to-date.
8654	*/
8655	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8656	{
8657	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8658
8659	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8660	hmR0VmxCheckExitDueToEventDelivery(). */
8661	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8662	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8663	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8664	AssertRCReturn(rc, rc);
8665	return rc;
8666	}
8667
8668
8669	/**
8670	* VM-exit exception handler for #PF (Page-fault exception).
8671	*/
8672	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8673	{
8674	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8675
8676	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8677	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8678	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8679	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8680	AssertRCReturn(rc, rc);
8681
8682	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8683	if (pVM->hm.s.fNestedPaging)
8684	{
8685	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8686	{
8687	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8688	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8689	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8690	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8691	AssertRCReturn(rc, rc);
8692	}
8693	else
8694	{
8695	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8696	Assert(!pVCpu->hm.s.Event.fPending);
8697	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8698	}
8699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8700	return rc;
8701	}
8702	#else
8703	Assert(!pVM->hm.s.fNestedPaging);
8704	#endif
8705
8706	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8707	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8708	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8709	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8710	AssertRCReturn(rc, rc);
8711	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8712	if ( pVM->hm.s.fTRPPatchingAllowed
8713	&& pVM->hm.s.pGuestPatchMem
8714	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8715	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8716	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8717	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8718	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8719	{
8720	RTGCPHYS GCPhys;
8721	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8722	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8723	if ( rc == VINF_SUCCESS
8724	&& GCPhys == GCPhysApicBase)
8725	{
8726	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8727	AssertRCReturn(rc, rc);
8728
8729	/* Only attempt to patch the instruction once. */
8730	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8731	if (!pPatch)
8732	return VINF_EM_HM_PATCH_TPR_INSTR;
8733	}
8734	}
8735	#endif
8736
8737	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8738	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8739	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8740
8741	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8742	AssertRCReturn(rc, rc);
8743
8744	/* Forward it to the trap handler first. */
8745	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8746	(RTGCPTR)pVmxTransient->uExitQualification);
8747
8748	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8749	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8750
8751	if (rc == VINF_SUCCESS)
8752	{
8753	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8754	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8755	* memory? We don't update the whole state here... */
8756	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8757	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8758	TRPMResetTrap(pVCpu);
8759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8760	return rc;
8761	}
8762	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8763	{
8764	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8765	{
8766	/* It's a guest page fault and needs to be reflected to the guest. */
8767	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8768	TRPMResetTrap(pVCpu);
8769	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8770	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8771	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8772	pVmxTransient->cbInstr, uGstErrorCode);
8773	AssertRCReturn(rc, rc);
8774	}
8775	else
8776	{
8777	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8778	Assert(!pVCpu->hm.s.Event.fPending);
8779	TRPMResetTrap(pVCpu);
8780	Log(("#PF: Injecting #DF\n"));
8781	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8782	}
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8784	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8785	return rc;
8786	}
8787
8788	TRPMResetTrap(pVCpu);
8789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8790	return rc;
8791	}
8792

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45481

Download in other formats: