HMVMXR0.cpp@ 45618

Last change on this file since 45618 was 45617, checked in by vboxsync, 12 years ago
VMMR0/HMVMXR0: Fix MOV DRx intercepts. Haiku now works.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 374.9 KB

Line
1	/* $Id: HMVMXR0.cpp 45617 2013-04-18 17:56:25Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_SAVE_FULL_VTX_STATE
39	#define VBOX_ALWAYS_SYNC_FULL_VTX_STATE
40	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
41	#endif
42
43	/*******************************************************************************
44	* Defined Constants And Macros *
45	*******************************************************************************/
46	#if defined(RT_ARCH_AMD64)
47	# define VMX_IS_64BIT_HOST_MODE() (true)
48	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
49	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
50	#else
51	# define VMX_IS_64BIT_HOST_MODE() (false)
52	#endif
53
54	#define VMX_SEL_UNUSABLE RT_BIT(16)
55	#define VMX_FLUSH_TAGGED_TLB_EPT_VPID 0
56	#define VMX_FLUSH_TAGGED_TLB_EPT 1
57	#define VMX_FLUSH_TAGGED_TLB_VPID 2
58	#define VMX_FLUSH_TAGGED_TLB_NONE 3
59
60	/**
61	* Updated-guest-state flags.
62	*/
63	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
64	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
65	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
66	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
67	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
68	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
69	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
70	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
71	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
72	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
73	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
74	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
75	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
76	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
77	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
78	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
79	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
80	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
81	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
82	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
83	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
84	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
85	\| VMX_UPDATED_GUEST_RIP \
86	\| VMX_UPDATED_GUEST_RSP \
87	\| VMX_UPDATED_GUEST_RFLAGS \
88	\| VMX_UPDATED_GUEST_CR0 \
89	\| VMX_UPDATED_GUEST_CR3 \
90	\| VMX_UPDATED_GUEST_CR4 \
91	\| VMX_UPDATED_GUEST_GDTR \
92	\| VMX_UPDATED_GUEST_IDTR \
93	\| VMX_UPDATED_GUEST_LDTR \
94	\| VMX_UPDATED_GUEST_TR \
95	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
96	\| VMX_UPDATED_GUEST_DEBUG \
97	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
98	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
99	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
100	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
101	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
102	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
103	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
104	\| VMX_UPDATED_GUEST_APIC_STATE)
105
106	/**
107	* Flags to skip redundant reads of some common VMCS fields.
108	*/
109	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
110	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
111	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
112	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
113	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
114	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
115
116	/**
117	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
118	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
119	* we have Nested Paging support.
120	*/
121	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
122	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
123	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
124	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
125	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
126	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
127	\| RT_BIT(X86_XCPT_XF))
128
129	/**
130	* Exception bitmap mask for all contributory exceptions.
131	*/
132	#define VMX_CONTRIBUTORY_XCPT_BITMAP ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
133	\| RT_BIT(X86_XCPT_DE))
134
135	/** Maximum VM-instruction error number. */
136	#define VMX_INSTR_ERROR_MAX 28
137
138	/*******************************************************************************
139	* Structures and Typedefs *
140	*******************************************************************************/
141	/**
142	* A state structure for holding miscellaneous information across
143	* VMX non-root operation and restored after the transition.
144	*/
145	typedef struct VMXTRANSIENT
146	{
147	/** The host's rflags/eflags. */
148	RTCCUINTREG uEFlags;
149	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
150	uint64_t u64LStarMsr;
151	/** The guest's TPR value used for TPR shadowing. */
152	uint8_t u8GuestTpr;
153
154	/** The basic VM-exit reason. */
155	uint16_t uExitReason;
156	/** The VM-exit exit qualification. */
157	RTGCUINTPTR uExitQualification;
158	/** The VM-exit interruption error code. */
159	uint32_t uExitIntrErrorCode;
160
161	/** The VM-exit interruption-information field. */
162	uint32_t uExitIntrInfo;
163	/** Whether the VM-entry failed or not. */
164	bool fVMEntryFailed;
165	/** The VM-exit instruction-length field. */
166	uint32_t cbInstr;
167
168	/** The VM-entry interruption-information field. */
169	uint32_t uEntryIntrInfo;
170	/** The VM-entry exception error code field. */
171	uint32_t uEntryXcptErrorCode;
172	/** The VM-entry instruction length field. */
173	uint32_t cbEntryInstr;
174
175	/** IDT-vectoring information field. */
176	uint32_t uIdtVectoringInfo;
177	/** IDT-vectoring error code. */
178	uint32_t uIdtVectoringErrorCode;
179
180	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
181	uint32_t fVmcsFieldsRead;
182	/** Whether TSC-offsetting should be setup before VM-entry. */
183	bool fUpdateTscOffsettingAndPreemptTimer;
184	/** Whether the VM-exit was caused by a page-fault during delivery of a
185	* contributary exception or a page-fault. */
186	bool fVectoringPF;
187	} VMXTRANSIENT, *PVMXTRANSIENT;
188
189	/**
190	* MSR-bitmap read permissions.
191	*/
192	typedef enum VMXMSREXITREAD
193	{
194	/** Reading this MSR causes a VM-exit. */
195	VMXMSREXIT_INTERCEPT_READ = 0xb,
196	/** Reading this MSR does not cause a VM-exit. */
197	VMXMSREXIT_PASSTHRU_READ
198	} VMXMSREXITREAD;
199
200	/**
201	* MSR-bitmap write permissions.
202	*/
203	typedef enum VMXMSREXITWRITE
204	{
205	/** Writing to this MSR causes a VM-exit. */
206	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
207	/** Writing to this MSR does not cause a VM-exit. */
208	VMXMSREXIT_PASSTHRU_WRITE
209	} VMXMSREXITWRITE;
210
211	/*******************************************************************************
212	* Internal Functions *
213	*******************************************************************************/
214	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
215	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
216	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
217	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
218	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
219	#endif
220	#if 1
221	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
222	#endif
223
224	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275
276	/*******************************************************************************
277	* Global Variables *
278	*******************************************************************************/
279	/**
280	* VM-exit handler.
281	*
282	* @returns VBox status code.
283	* @param pVCpu Pointer to the VMCPU.
284	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
285	* out-of-sync. Make sure to update the required
286	* fields before using them.
287	* @param pVmxTransient Pointer to the VMX-transient structure.
288	*/
289	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	/** Pointer to VM-exit handler. */
291	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
292
293	/**
294	* VMX_EXIT dispatch table.
295	*/
296	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
297	{
298	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
299	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
300	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
301	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
302	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
303	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
304	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
305	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
306	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
307	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
308	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
309	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
310	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
311	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
312	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
313	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
314	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
315	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
316	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
317	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
318	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
319	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
320	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
321	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
322	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
323	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
324	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
325	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
326	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
327	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
328	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
329	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
330	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
331	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
332	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
333	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
334	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
335	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
336	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
337	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
338	/* 40 UNDEFINED */ hmR0VmxExitPause,
339	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
340	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
341	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
342	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
343	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
344	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
345	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
346	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
347	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
348	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
349	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
350	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
351	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
352	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
353	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
354	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
355	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
356	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
357	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
358	};
359
360	#ifdef VBOX_STRICT
361	static const char* const g_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
362	{
363	/* 0 */ "(Not Used)",
364	/* 1 */ "VMCALL executed in VMX root operation.",
365	/* 2 */ "VMCLEAR with invalid physical address.",
366	/* 3 */ "VMCLEAR with VMXON pointer.",
367	/* 4 */ "VMLAUNCH with non-clear VMCS.",
368	/* 5 */ "VMRESUME with non-launched VMCS.",
369	/* 6 */ "VMRESUME after VMXOFF",
370	/* 7 */ "VM entry with invalid control fields.",
371	/* 8 */ "VM entry with invalid host state fields.",
372	/* 9 */ "VMPTRLD with invalid physical address.",
373	/* 10 */ "VMPTRLD with VMXON pointer.",
374	/* 11 */ "VMPTRLD with incorrect revision identifier.",
375	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
376	/* 13 */ "VMWRITE to read-only VMCS component.",
377	/* 14 */ "(Not Used)",
378	/* 15 */ "VMXON executed in VMX root operation.",
379	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
380	/* 17 */ "VM entry with non-launched executing VMCS.",
381	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
382	/* 19 */ "VMCALL with non-clear VMCS.",
383	/* 20 */ "VMCALL with invalid VM-exit control fields.",
384	/* 21 */ "(Not Used)",
385	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
386	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
387	/* 24 */ "VMCALL with invalid SMM-monitor features.",
388	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
389	/* 26 */ "VM entry with events blocked by MOV SS.",
390	/* 27 */ "(Not Used)",
391	/* 28 */ "Invalid operand to INVEPT/INVVPID."
392	};
393	#endif
394
395	/**
396	* Updates the VM's last error record. If there was a VMX instruction error,
397	* reads the error data from the VMCS and updates VCPU's last error record as
398	* well.
399	*
400	* @param pVM Pointer to the VM.
401	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
402	* VERR_VMX_UNABLE_TO_START_VM or
403	* VERR_VMX_INVALID_VMCS_FIELD).
404	* @param rc The error code.
405	*/
406	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
407	{
408	AssertPtr(pVM);
409	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
410	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
411	{
412	AssertPtrReturnVoid(pVCpu);
413	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
414	}
415	pVM->hm.s.lLastError = rc;
416	}
417
418
419	/**
420	* Reads the VM-entry interruption-information field from the VMCS into the VMX
421	* transient structure.
422	*
423	* @returns VBox status code.
424	* @param pVmxTransient Pointer to the VMX transient structure.
425	*
426	* @remarks No-long-jump zone!!!
427	*/
428	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
429	{
430	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
431	AssertRCReturn(rc, rc);
432	return VINF_SUCCESS;
433	}
434
435
436	/**
437	* Reads the VM-entry exception error code field from the VMCS into
438	* the VMX transient structure.
439	*
440	* @returns VBox status code.
441	* @param pVmxTransient Pointer to the VMX transient structure.
442	*
443	* @remarks No-long-jump zone!!!
444	*/
445	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
446	{
447	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
448	AssertRCReturn(rc, rc);
449	return VINF_SUCCESS;
450	}
451
452
453	/**
454	* Reads the VM-entry exception error code field from the VMCS into
455	* the VMX transient structure.
456	*
457	* @returns VBox status code.
458	* @param pVCpu Pointer to the VMCPU.
459	* @param pVmxTransient Pointer to the VMX transient structure.
460	*
461	* @remarks No-long-jump zone!!!
462	*/
463	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
464	{
465	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
466	AssertRCReturn(rc, rc);
467	return VINF_SUCCESS;
468	}
469
470
471	/**
472	* Reads the VM-exit interruption-information field from the VMCS into the VMX
473	* transient structure.
474	*
475	* @returns VBox status code.
476	* @param pVCpu Pointer to the VMCPU.
477	* @param pVmxTransient Pointer to the VMX transient structure.
478	*/
479	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
480	{
481	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
482	{
483	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
484	AssertRCReturn(rc, rc);
485	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
486	}
487	return VINF_SUCCESS;
488	}
489
490
491	/**
492	* Reads the VM-exit interruption error code from the VMCS into the VMX
493	* transient structure.
494	*
495	* @returns VBox status code.
496	* @param pVCpu Pointer to the VMCPU.
497	* @param pVmxTransient Pointer to the VMX transient structure.
498	*/
499	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
500	{
501	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
502	{
503	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
504	AssertRCReturn(rc, rc);
505	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
506	}
507	return VINF_SUCCESS;
508	}
509
510
511	/**
512	* Reads the VM-exit instruction length field from the VMCS into the VMX
513	* transient structure.
514	*
515	* @returns VBox status code.
516	* @param pVCpu Pointer to the VMCPU.
517	* @param pVmxTransient Pointer to the VMX transient structure.
518	*/
519	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
520	{
521	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
522	{
523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
524	AssertRCReturn(rc, rc);
525	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
526	}
527	return VINF_SUCCESS;
528	}
529
530
531	/**
532	* Reads the exit qualification from the VMCS into the VMX transient structure.
533	*
534	* @returns VBox status code.
535	* @param pVCpu Pointer to the VMCPU.
536	* @param pVmxTransient Pointer to the VMX transient structure.
537	*/
538	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
539	{
540	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
541	{
542	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
543	AssertRCReturn(rc, rc);
544	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
545	}
546	return VINF_SUCCESS;
547	}
548
549
550	/**
551	* Reads the IDT-vectoring information field from the VMCS into the VMX
552	* transient structure.
553	*
554	* @returns VBox status code.
555	* @param pVmxTransient Pointer to the VMX transient structure.
556	*
557	* @remarks No-long-jump zone!!!
558	*/
559	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
560	{
561	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
562	{
563	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
564	AssertRCReturn(rc, rc);
565	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
566	}
567	return VINF_SUCCESS;
568	}
569
570
571	/**
572	* Reads the IDT-vectoring error code from the VMCS into the VMX
573	* transient structure.
574	*
575	* @returns VBox status code.
576	* @param pVmxTransient Pointer to the VMX transient structure.
577	*/
578	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
579	{
580	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
581	{
582	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
583	AssertRCReturn(rc, rc);
584	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
585	}
586	return VINF_SUCCESS;
587	}
588
589
590	/**
591	* Enters VMX root mode operation on the current CPU.
592	*
593	* @returns VBox status code.
594	* @param pVM Pointer to the VM (optional, can be NULL, after
595	* a resume).
596	* @param HCPhysCpuPage Physical address of the VMXON region.
597	* @param pvCpuPage Pointer to the VMXON region.
598	*/
599	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
600	{
601	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
602	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
603
604	if (pVM)
605	{
606	/* Write the VMCS revision dword to the VMXON region. */
607	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
608	}
609
610	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
611	RTCCUINTREG fFlags = ASMIntDisableFlags();
612
613	/* Enable the VMX bit in CR4 if necessary. */
614	RTCCUINTREG uCr4 = ASMGetCR4();
615	if (!(uCr4 & X86_CR4_VMXE))
616	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
617
618	/* Enter VMX root mode. */
619	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
620	if (RT_FAILURE(rc))
621	ASMSetCR4(uCr4);
622
623	/* Restore interrupts. */
624	ASMSetFlags(fFlags);
625	return rc;
626	}
627
628
629	/**
630	* Exits VMX root mode operation on the current CPU.
631	*
632	* @returns VBox status code.
633	*/
634	static int hmR0VmxLeaveRootMode(void)
635	{
636	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
637	RTCCUINTREG fFlags = ASMIntDisableFlags();
638	int rc = VINF_SUCCESS;
639
640	/* If we're for some reason not in VMX root mode, then don't leave it. */
641	if (ASMGetCR4() & X86_CR4_VMXE)
642	{
643	/* Exit VMX root mode and clear the VMX bit in CR4 */
644	VMXDisable();
645	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
646	}
647	else
648	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
649
650	/* Restore interrupts. */
651	ASMSetFlags(fFlags);
652	return rc;
653	}
654
655
656	/**
657	* Allocates and maps one physically contiguous page. The allocated page is
658	* zero'd out. (Used by various VT-x structures).
659	*
660	* @returns IPRT status code.
661	* @param pMemObj Pointer to the ring-0 memory object.
662	* @param ppVirt Where to store the virtual address of the
663	* allocation.
664	* @param pPhys Where to store the physical address of the
665	* allocation.
666	*/
667	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
668	{
669	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
670	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
671	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
672
673	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
674	if (RT_FAILURE(rc))
675	return rc;
676	ppVirt = RTR0MemObjAddress(pMemObj);
677	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
678	ASMMemZero32(*ppVirt, PAGE_SIZE);
679	return VINF_SUCCESS;
680	}
681
682
683	/**
684	* Frees and unmaps an allocated physical page.
685	*
686	* @param pMemObj Pointer to the ring-0 memory object.
687	* @param ppVirt Where to re-initialize the virtual address of
688	* allocation as 0.
689	* @param pHCPhys Where to re-initialize the physical address of the
690	* allocation as 0.
691	*/
692	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
693	{
694	AssertPtr(pMemObj);
695	AssertPtr(ppVirt);
696	AssertPtr(pHCPhys);
697	if (*pMemObj != NIL_RTR0MEMOBJ)
698	{
699	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
700	AssertRC(rc);
701	*pMemObj = NIL_RTR0MEMOBJ;
702	*ppVirt = 0;
703	*pHCPhys = 0;
704	}
705	}
706
707
708	/**
709	* Worker function to free VT-x related structures.
710	*
711	* @returns IPRT status code.
712	* @param pVM Pointer to the VM.
713	*/
714	static void hmR0VmxStructsFree(PVM pVM)
715	{
716	for (VMCPUID i = 0; i < pVM->cCpus; i++)
717	{
718	PVMCPU pVCpu = &pVM->aCpus[i];
719	AssertPtr(pVCpu);
720
721	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
722	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
723
724	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
726
727	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
728	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
729	}
730
731	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
732	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
733	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
734	#endif
735	}
736
737
738	/**
739	* Worker function to allocate VT-x related VM structures.
740	*
741	* @returns IPRT status code.
742	* @param pVM Pointer to the VM.
743	*/
744	static int hmR0VmxStructsAlloc(PVM pVM)
745	{
746	/*
747	* Initialize members up-front so we can cleanup properly on allocation failure.
748	*/
749	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
750	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
751	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
752	pVM->hm.s.vmx.HCPhys##a_Name = 0;
753
754	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
755	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
756	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
757	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
758
759	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
760	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
761	#endif
762	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
763
764	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
765	for (VMCPUID i = 0; i < pVM->cCpus; i++)
766	{
767	PVMCPU pVCpu = &pVM->aCpus[i];
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
770	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
771	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
772	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
773	}
774	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
775	#undef VMXLOCAL_INIT_VM_MEMOBJ
776
777	/*
778	* Allocate all the VT-x structures.
779	*/
780	int rc = VINF_SUCCESS;
781	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
782	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
783	if (RT_FAILURE(rc))
784	goto cleanup;
785	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
786	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
787	#endif
788
789	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
790	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
791	{
792	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
793	&pVM->hm.s.vmx.HCPhysApicAccess);
794	if (RT_FAILURE(rc))
795	goto cleanup;
796	}
797
798	/*
799	* Initialize per-VCPU VT-x structures.
800	*/
801	for (VMCPUID i =0; i < pVM->cCpus; i++)
802	{
803	PVMCPU pVCpu = &pVM->aCpus[i];
804	AssertPtr(pVCpu);
805
806	/* Allocate the VM control structure (VMCS). */
807	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
808	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
809	if (RT_FAILURE(rc))
810	goto cleanup;
811
812	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
813	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
814	{
815	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
816	&pVCpu->hm.s.vmx.HCPhysVirtApic);
817	if (RT_FAILURE(rc))
818	goto cleanup;
819	}
820
821	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
822	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
823	{
824	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
825	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
826	if (RT_FAILURE(rc))
827	goto cleanup;
828	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
829	}
830
831	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
832	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
833	if (RT_FAILURE(rc))
834	goto cleanup;
835
836	/* Allocate the VM-exit MSR-load page for the host MSRs. */
837	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840	}
841
842	return VINF_SUCCESS;
843
844	cleanup:
845	hmR0VmxStructsFree(pVM);
846	return rc;
847	}
848
849
850	/**
851	* Does global VT-x initialization (called during module initialization).
852	*
853	* @returns VBox status code.
854	*/
855	VMMR0DECL(int) VMXR0GlobalInit(void)
856	{
857	/* Setup the main VM exit handlers. */
858	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
859	#ifdef VBOX_STRICT
860	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
861	Assert(g_apfnVMExitHandlers[i]);
862	#endif
863	return VINF_SUCCESS;
864	}
865
866
867	/**
868	* Does global VT-x termination (called during module termination).
869	*/
870	VMMR0DECL(void) VMXR0GlobalTerm()
871	{
872	/* Nothing to do currently. */
873	}
874
875
876	/**
877	* Sets up and activates VT-x on the current CPU.
878	*
879	* @returns VBox status code.
880	* @param pCpu Pointer to the global CPU info struct.
881	* @param pVM Pointer to the VM (can be NULL after a host resume
882	* operation).
883	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
884	* fEnabledByHost is true).
885	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
886	* @a fEnabledByHost is true).
887	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
888	* enable VT-x/AMD-V on the host.
889	*/
890	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
891	{
892	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
893	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
894
895	if (!fEnabledByHost)
896	{
897	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
898	if (RT_FAILURE(rc))
899	return rc;
900	}
901
902	/*
903	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
904	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
905	* each time while reusing a VPID after hitting the MaxASID limit once.
906	*/
907	if ( pVM
908	&& pVM->hm.s.vmx.fVpid
909	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
910	{
911	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
912	pCpu->fFlushAsidBeforeUse = false;
913	}
914	else
915	pCpu->fFlushAsidBeforeUse = true;
916
917	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
918	++pCpu->cTlbFlushes;
919
920	return VINF_SUCCESS;
921	}
922
923
924	/**
925	* Deactivates VT-x on the current CPU.
926	*
927	* @returns VBox status code.
928	* @param pCpu Pointer to the global CPU info struct.
929	* @param pvCpuPage Pointer to the VMXON region.
930	* @param HCPhysCpuPage Physical address of the VMXON region.
931	*/
932	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
933	{
934	NOREF(pCpu);
935	NOREF(pvCpuPage);
936	NOREF(HCPhysCpuPage);
937
938	hmR0VmxLeaveRootMode();
939	return VINF_SUCCESS;
940	}
941
942
943	/**
944	* Sets the permission bits for the specified MSR in the MSR bitmap.
945	*
946	* @param pVCpu Pointer to the VMCPU.
947	* @param uMSR The MSR value.
948	* @param enmRead Whether reading this MSR causes a VM-exit.
949	* @param enmWrite Whether writing this MSR causes a VM-exit.
950	*/
951	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
952	{
953	int32_t iBit;
954	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
955
956	/*
957	* Layout:
958	* 0x000 - 0x3ff - Low MSR read bits
959	* 0x400 - 0x7ff - High MSR read bits
960	* 0x800 - 0xbff - Low MSR write bits
961	* 0xc00 - 0xfff - High MSR write bits
962	*/
963	if (uMsr <= 0x00001FFF)
964	{
965	/* Pentium-compatible MSRs */
966	iBit = uMsr;
967	}
968	else if ( uMsr >= 0xC0000000
969	&& uMsr <= 0xC0001FFF)
970	{
971	/* AMD Sixth Generation x86 Processor MSRs */
972	iBit = (uMsr - 0xC0000000);
973	pbMsrBitmap += 0x400;
974	}
975	else
976	{
977	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
978	return;
979	}
980
981	Assert(iBit <= 0x1fff);
982	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
983	ASMBitSet(pbMsrBitmap, iBit);
984	else
985	ASMBitClear(pbMsrBitmap, iBit);
986
987	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
988	ASMBitSet(pbMsrBitmap + 0x800, iBit);
989	else
990	ASMBitClear(pbMsrBitmap + 0x800, iBit);
991	}
992
993
994	/**
995	* Flushes the TLB using EPT.
996	*
997	* @returns VBox status code.
998	* @param pVM Pointer to the VM.
999	* @param pVCpu Pointer to the VMCPU.
1000	* @param enmFlush Type of flush.
1001	*/
1002	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1003	{
1004	AssertPtr(pVM);
1005	Assert(pVM->hm.s.fNestedPaging);
1006
1007	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1008
1009	uint64_t descriptor[2];
1010	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1011	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1012
1013	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1014	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1015	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1016	}
1017
1018
1019	/**
1020	* Flushes the TLB using VPID.
1021	*
1022	* @returns VBox status code.
1023	* @param pVM Pointer to the VM.
1024	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1025	* enmFlush).
1026	* @param enmFlush Type of flush.
1027	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1028	* on @a enmFlush).
1029	*/
1030	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.vmx.fVpid);
1034
1035	uint64_t descriptor[2];
1036	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1037	{
1038	descriptor[0] = 0;
1039	descriptor[1] = 0;
1040	}
1041	else
1042	{
1043	AssertPtr(pVCpu);
1044	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1045	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1046	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1047	descriptor[1] = GCPtr;
1048	}
1049
1050	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1051	AssertMsg(rc == VINF_SUCCESS,
1052	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1053	if ( RT_SUCCESS(rc)
1054	&& pVCpu)
1055	{
1056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1057	}
1058	}
1059
1060
1061	/**
1062	* Invalidates a guest page by guest virtual address. Only relevant for
1063	* EPT/VPID, otherwise there is nothing really to invalidate.
1064	*
1065	* @returns VBox status code.
1066	* @param pVM Pointer to the VM.
1067	* @param pVCpu Pointer to the VMCPU.
1068	* @param GCVirt Guest virtual address of the page to invalidate.
1069	*/
1070	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1071	{
1072	AssertPtr(pVM);
1073	AssertPtr(pVCpu);
1074	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1075
1076	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1077	if (!fFlushPending)
1078	{
1079	/*
1080	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1081	* See @bugref{6043} and @bugref{6177}.
1082	*
1083	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1084	* function maybe called in a loop with individual addresses.
1085	*/
1086	if (pVM->hm.s.vmx.fVpid)
1087	{
1088	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1089	{
1090	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1091	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1092	}
1093	else
1094	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1095	}
1096	else if (pVM->hm.s.fNestedPaging)
1097	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1098	}
1099
1100	return VINF_SUCCESS;
1101	}
1102
1103
1104	/**
1105	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1106	* otherwise there is nothing really to invalidate.
1107	*
1108	* @returns VBox status code.
1109	* @param pVM Pointer to the VM.
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param GCPhys Guest physical address of the page to invalidate.
1112	*/
1113	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1114	{
1115	LogFlowFunc(("%RGp\n", GCPhys));
1116
1117	/*
1118	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1119	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1120	* This function might be called in a loop.
1121	*/
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1130	* case where neither EPT nor VPID is supported by the CPU.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	*
1135	* @remarks Called with interrupts disabled.
1136	*/
1137	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1138	{
1139	NOREF(pVM);
1140	AssertPtr(pVCpu);
1141	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1143
1144	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1145	AssertPtr(pCpu);
1146
1147	pVCpu->hm.s.TlbShootdown.cPages = 0;
1148	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1149	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1150	pVCpu->hm.s.fForceTLBFlush = false;
1151	return;
1152	}
1153
1154
1155	/**
1156	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1157	*
1158	* @param pVM Pointer to the VM.
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @remarks All references to "ASID" in this function pertains to "VPID" in
1161	* Intel's nomenclature. The reason is, to avoid confusion in compare
1162	* statements since the host-CPU copies are named "ASID".
1163	*
1164	* @remarks Called with interrupts disabled.
1165	*/
1166	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1167	{
1168	AssertPtr(pVM);
1169	AssertPtr(pVCpu);
1170	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1171	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1172	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1173
1174	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1175	AssertPtr(pCpu);
1176
1177	/*
1178	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1179	* This can happen both for start & resume due to long jumps back to ring-3.
1180	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1181	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1182	*/
1183	bool fNewASID = false;
1184	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1185	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1186	{
1187	pVCpu->hm.s.fForceTLBFlush = true;
1188	fNewASID = true;
1189	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1190	}
1191
1192	/*
1193	* Check for explicit TLB shootdowns.
1194	*/
1195	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1196	{
1197	pVCpu->hm.s.fForceTLBFlush = true;
1198	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1199	}
1200
1201	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1202	if (pVCpu->hm.s.fForceTLBFlush)
1203	{
1204	if (fNewASID)
1205	{
1206	++pCpu->uCurrentAsid;
1207	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1208	{
1209	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1210	pCpu->cTlbFlushes++;
1211	pCpu->fFlushAsidBeforeUse = true;
1212	}
1213
1214	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1215	if (pCpu->fFlushAsidBeforeUse)
1216	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1217	}
1218	else
1219	{
1220	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1221	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1222	else
1223	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1224	}
1225
1226	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1227	pVCpu->hm.s.fForceTLBFlush = false;
1228	}
1229	else
1230	{
1231	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1232	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1233	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1234	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1235
1236	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1237	* not be executed. See hmQueueInvlPage() where it is commented
1238	* out. Support individual entry flushing someday. */
1239	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1240	{
1241	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1242
1243	/*
1244	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1245	* as supported by the CPU.
1246	*/
1247	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1248	{
1249	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1250	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1251	}
1252	else
1253	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1254	}
1255	else
1256	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1257	}
1258	pVCpu->hm.s.TlbShootdown.cPages = 0;
1259	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1260
1261	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1262	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1263	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1264	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1265	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1266	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1267
1268	/* Update VMCS with the VPID. */
1269	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1270	AssertRC(rc);
1271	}
1272
1273
1274	/**
1275	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1276	*
1277	* @returns VBox status code.
1278	* @param pVM Pointer to the VM.
1279	* @param pVCpu Pointer to the VMCPU.
1280	*
1281	* @remarks Called with interrupts disabled.
1282	*/
1283	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1284	{
1285	AssertPtr(pVM);
1286	AssertPtr(pVCpu);
1287	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1288	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1289
1290	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1291	AssertPtr(pCpu);
1292
1293	/*
1294	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1295	* This can happen both for start & resume due to long jumps back to ring-3.
1296	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1297	*/
1298	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1299	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1300	{
1301	pVCpu->hm.s.fForceTLBFlush = true;
1302	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1303	}
1304
1305	/* Check for explicit TLB shootdown flushes. */
1306	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1307	{
1308	pVCpu->hm.s.fForceTLBFlush = true;
1309	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1310	}
1311
1312	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1313	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1314
1315	if (pVCpu->hm.s.fForceTLBFlush)
1316	{
1317	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1318	pVCpu->hm.s.fForceTLBFlush = false;
1319	}
1320	else
1321	{
1322	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1323	* not be executed. See hmQueueInvlPage() where it is commented
1324	* out. Support individual entry flushing someday. */
1325	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1326	{
1327	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1328	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1329	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1330	}
1331	else
1332	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1333	}
1334
1335	pVCpu->hm.s.TlbShootdown.cPages = 0;
1336	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1337	}
1338
1339
1340	/**
1341	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1342	*
1343	* @returns VBox status code.
1344	* @param pVM Pointer to the VM.
1345	* @param pVCpu Pointer to the VMCPU.
1346	*
1347	* @remarks Called with interrupts disabled.
1348	*/
1349	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1350	{
1351	AssertPtr(pVM);
1352	AssertPtr(pVCpu);
1353	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1354	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1355
1356	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1357
1358	/*
1359	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1360	* This can happen both for start & resume due to long jumps back to ring-3.
1361	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1362	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1363	*/
1364	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1365	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1366	{
1367	pVCpu->hm.s.fForceTLBFlush = true;
1368	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1369	}
1370
1371	/* Check for explicit TLB shootdown flushes. */
1372	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1373	{
1374	/*
1375	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1376	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1377	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1378	*/
1379	pVCpu->hm.s.fForceTLBFlush = true;
1380	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1381	}
1382
1383	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1384	if (pVCpu->hm.s.fForceTLBFlush)
1385	{
1386	++pCpu->uCurrentAsid;
1387	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1388	{
1389	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1390	pCpu->fFlushAsidBeforeUse = true;
1391	pCpu->cTlbFlushes++;
1392	}
1393
1394	pVCpu->hm.s.fForceTLBFlush = false;
1395	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1396	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1397	if (pCpu->fFlushAsidBeforeUse)
1398	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1399	}
1400	else
1401	{
1402	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1403	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1404	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1405	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1406
1407	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1408	* not be executed. See hmQueueInvlPage() where it is commented
1409	* out. Support individual entry flushing someday. */
1410	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1411	{
1412	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1413	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1414	{
1415	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1416	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1417	}
1418	else
1419	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1420	}
1421	else
1422	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1423	}
1424
1425	pVCpu->hm.s.TlbShootdown.cPages = 0;
1426	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1427
1428	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1429	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1430	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1431	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1432	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1433	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1434
1435	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1436	AssertRC(rc);
1437	}
1438
1439
1440	/**
1441	* Flushes the guest TLB entry based on CPU capabilities.
1442	*
1443	* @param pVCpu Pointer to the VMCPU.
1444	*/
1445	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1446	{
1447	PVM pVM = pVCpu->CTX_SUFF(pVM);
1448	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1449	{
1450	case VMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1451	case VMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1452	case VMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1453	case VMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1454	default:
1455	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1456	break;
1457	}
1458	}
1459
1460
1461	/**
1462	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1463	* TLB entries from the host TLB before VM-entry.
1464	*
1465	* @returns VBox status code.
1466	* @param pVM Pointer to the VM.
1467	*/
1468	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1469	{
1470	/*
1471	* Determine optimal flush type for nested paging.
1472	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1473	* guest execution (see hmR3InitFinalizeR0()).
1474	*/
1475	if (pVM->hm.s.fNestedPaging)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1478	{
1479	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1480	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1481	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1482	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1483	else
1484	{
1485	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1486	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1487	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1488	}
1489
1490	/* Make sure the write-back cacheable memory type for EPT is supported. */
1491	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1492	{
1493	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1494	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1495	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1496	}
1497	}
1498	else
1499	{
1500	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1501	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1502	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1503	}
1504	}
1505
1506	/*
1507	* Determine optimal flush type for VPID.
1508	*/
1509	if (pVM->hm.s.vmx.fVpid)
1510	{
1511	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1512	{
1513	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1515	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1516	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1517	else
1518	{
1519	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1520	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1521	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1522	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1523	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1524	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1525	pVM->hm.s.vmx.fVpid = false;
1526	}
1527	}
1528	else
1529	{
1530	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1531	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1532	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1533	pVM->hm.s.vmx.fVpid = false;
1534	}
1535	}
1536
1537	/*
1538	* Setup the handler for flushing tagged-TLBs.
1539	*/
1540	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1541	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT_VPID;
1542	else if (pVM->hm.s.fNestedPaging)
1543	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT;
1544	else if (pVM->hm.s.vmx.fVpid)
1545	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_VPID;
1546	else
1547	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_NONE;
1548	return VINF_SUCCESS;
1549	}
1550
1551
1552	/**
1553	* Sets up pin-based VM-execution controls in the VMCS.
1554	*
1555	* @returns VBox status code.
1556	* @param pVM Pointer to the VM.
1557	* @param pVCpu Pointer to the VMCPU.
1558	*/
1559	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1560	{
1561	AssertPtr(pVM);
1562	AssertPtr(pVCpu);
1563
1564	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1565	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1566
1567	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1568	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1569	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1570
1571	/* Enable the VMX preemption timer. */
1572	if (pVM->hm.s.vmx.fUsePreemptTimer)
1573	{
1574	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1575	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1576	}
1577
1578	if ((val & zap) != val)
1579	{
1580	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1581	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1582	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1583	}
1584
1585	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1586	AssertRCReturn(rc, rc);
1587
1588	/* Update VCPU with the currently set pin-based VM-execution controls. */
1589	pVCpu->hm.s.vmx.u32PinCtls = val;
1590	return rc;
1591	}
1592
1593
1594	/**
1595	* Sets up processor-based VM-execution controls in the VMCS.
1596	*
1597	* @returns VBox status code.
1598	* @param pVM Pointer to the VM.
1599	* @param pVMCPU Pointer to the VMCPU.
1600	*/
1601	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1602	{
1603	AssertPtr(pVM);
1604	AssertPtr(pVCpu);
1605
1606	int rc = VERR_INTERNAL_ERROR_5;
1607	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1608	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1609
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1613	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1614	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1615	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1616	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1617
1618	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1619	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1620	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1621	{
1622	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1623	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1624	}
1625
1626	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1627	if (!pVM->hm.s.fNestedPaging)
1628	{
1629	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1630	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1632	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1633	}
1634
1635	/* Use TPR shadowing if supported by the CPU. */
1636	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1637	{
1638	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1639	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1640	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1641	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1642	AssertRCReturn(rc, rc);
1643
1644	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1645	/* CR8 writes causes a VM-exit based on TPR threshold. */
1646	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1647	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1648	}
1649	else
1650	{
1651	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1653	}
1654
1655	/* Use MSR-bitmaps if supported by the CPU. */
1656	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1657	{
1658	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1659
1660	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1661	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1662	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1663	AssertRCReturn(rc, rc);
1664
1665	/*
1666	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1667	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1668	*/
1669	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1670	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1671	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1672	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1673	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1674	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1675	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1676	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1677	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1678	}
1679
1680	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1681	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1682	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1683
1684	if ((val & zap) != val)
1685	{
1686	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1687	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1688	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1689	}
1690
1691	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1692	AssertRCReturn(rc, rc);
1693
1694	/* Update VCPU with the currently set processor-based VM-execution controls. */
1695	pVCpu->hm.s.vmx.u32ProcCtls = val;
1696
1697	/*
1698	* Secondary processor-based VM-execution controls.
1699	*/
1700	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1701	{
1702	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1703	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1704
1705	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1706	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1707
1708	if (pVM->hm.s.fNestedPaging)
1709	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1710	else
1711	{
1712	/*
1713	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1714	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1715	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1716	*/
1717	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1718	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1719	}
1720
1721	if (pVM->hm.s.vmx.fVpid)
1722	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1723
1724	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1725	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1726
1727	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1728	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1729	* done dynamically. */
1730	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1731	{
1732	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1733	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1734	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1735	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1736	AssertRCReturn(rc, rc);
1737	}
1738
1739	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1740	{
1741	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1742	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1743	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1744	}
1745
1746	if ((val & zap) != val)
1747	{
1748	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1749	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1750	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1751	}
1752
1753	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1754	AssertRCReturn(rc, rc);
1755
1756	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1757	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1758	}
1759
1760	return VINF_SUCCESS;
1761	}
1762
1763
1764	/**
1765	* Sets up miscellaneous (everything other than Pin & Processor-based
1766	* VM-execution) control fields in the VMCS.
1767	*
1768	* @returns VBox status code.
1769	* @param pVM Pointer to the VM.
1770	* @param pVCpu Pointer to the VMCPU.
1771	*/
1772	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1773	{
1774	AssertPtr(pVM);
1775	AssertPtr(pVCpu);
1776
1777	int rc = VERR_GENERAL_FAILURE;
1778
1779	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1781
1782	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1783
1784	/*
1785	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1786	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1787	* We thus use the exception bitmap to control it rather than use both.
1788	*/
1789	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1790	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1791
1792	/** @todo Explore possibility of using IO-bitmaps. */
1793	/* All IO & IOIO instructions cause VM-exits. */
1794	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1795	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1796
1797	/* Setup MSR autoloading/autostoring. */
1798	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1799	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1800	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1801	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1802	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1803	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1804
1805	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1806	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1807	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1808	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1809
1810	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1811	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1812
1813	/* Setup debug controls */
1814	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1815	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1816	AssertRCReturn(rc, rc);
1817	return rc;
1818	}
1819
1820
1821	/**
1822	* Sets up the initial exception bitmap in the VMCS based on static conditions
1823	* (i.e. conditions that cannot ever change at runtime).
1824	*
1825	* @returns VBox status code.
1826	* @param pVM Pointer to the VM.
1827	* @param pVCpu Pointer to the VMCPU.
1828	*/
1829	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1830	{
1831	AssertPtr(pVM);
1832	AssertPtr(pVCpu);
1833
1834	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1835
1836	uint32_t u32XcptBitmap = 0;
1837
1838	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1839	if (!pVM->hm.s.fNestedPaging)
1840	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1841
1842	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1843	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1844	AssertRCReturn(rc, rc);
1845	return rc;
1846	}
1847
1848
1849	/**
1850	* Sets up the initial guest-state mask. The guest-state mask is consulted
1851	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1852	* for the nested virtualization case (as it would cause a VM-exit).
1853	*
1854	* @param pVCpu Pointer to the VMCPU.
1855	*/
1856	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1857	{
1858	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1859	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1860	return VINF_SUCCESS;
1861	}
1862
1863
1864	/**
1865	* Does per-VM VT-x initialization.
1866	*
1867	* @returns VBox status code.
1868	* @param pVM Pointer to the VM.
1869	*/
1870	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1871	{
1872	LogFlowFunc(("pVM=%p\n", pVM));
1873
1874	int rc = hmR0VmxStructsAlloc(pVM);
1875	if (RT_FAILURE(rc))
1876	{
1877	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1878	return rc;
1879	}
1880
1881	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1882	{
1883	PVMCPU pVCpu = &pVM->aCpus[i];
1884
1885	/* Current guest paging mode. */
1886	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1887	}
1888
1889	return VINF_SUCCESS;
1890	}
1891
1892
1893	/**
1894	* Does per-VM VT-x termination.
1895	*
1896	* @returns VBox status code.
1897	* @param pVM Pointer to the VM.
1898	*/
1899	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1900	{
1901	LogFlowFunc(("pVM=%p\n", pVM));
1902
1903	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1904	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1905	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1906	#endif
1907	hmR0VmxStructsFree(pVM);
1908	return VINF_SUCCESS;
1909	}
1910
1911
1912	/**
1913	* Sets up the VM for execution under VT-x.
1914	* This function is only called once per-VM during initalization.
1915	*
1916	* @returns VBox status code.
1917	* @param pVM Pointer to the VM.
1918	*/
1919	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1920	{
1921	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1922	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1923
1924	LogFlowFunc(("pVM=%p\n", pVM));
1925
1926	/*
1927	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1928	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1929	*/
1930	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1931	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1932	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1933	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1934	{
1935	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1936	return VERR_INTERNAL_ERROR;
1937	}
1938
1939	/* Initialize these always, see hmR3InitFinalizeR0().*/
1940	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1941	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1942
1943	/* Setup the tagged-TLB flush handlers. */
1944	int rc = hmR0VmxSetupTaggedTlb(pVM);
1945	if (RT_FAILURE(rc))
1946	{
1947	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1948	return rc;
1949	}
1950
1951	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1952	{
1953	PVMCPU pVCpu = &pVM->aCpus[i];
1954	AssertPtr(pVCpu);
1955	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1956
1957	/* Set revision dword at the beginning of the VMCS structure. */
1958	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1959
1960	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1961	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1962	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1963	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1964
1965	/* Load this VMCS as the current VMCS. */
1966	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1967	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1968	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1969
1970	/* Setup the pin-based VM-execution controls. */
1971	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1972	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1973	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1974
1975	/* Setup the processor-based VM-execution controls. */
1976	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1977	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1978	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1979
1980	/* Setup the rest (miscellaneous) VM-execution controls. */
1981	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1982	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1983	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1984
1985	/* Setup the initial exception bitmap. */
1986	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1987	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1988	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1989
1990	/* Setup the initial guest-state mask. */
1991	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1992	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1993	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1994
1995	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1996	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1997	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000	#endif
2001
2002	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2003	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2004	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2005	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2006
2007	/* Update the last error record for this VCPU. */
2008	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2009	}
2010
2011	return VINF_SUCCESS;
2012	}
2013
2014
2015	/**
2016	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2017	* the VMCS.
2018	*
2019	* @returns VBox status code.
2020	* @param pVM Pointer to the VM.
2021	* @param pVCpu Pointer to the VMCPU.
2022	*/
2023	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2024	{
2025	RTCCUINTREG uReg = ASMGetCR0();
2026	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2027
2028	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2029	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2030	if (VMX_IS_64BIT_HOST_MODE())
2031	{
2032	uint64_t uReg = hmR0Get64bitCR3();
2033	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2034	}
2035	else
2036	#endif
2037	{
2038	uReg = ASMGetCR3();
2039	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2040	}
2041
2042	uReg = ASMGetCR4();
2043	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2044	AssertRCReturn(rc, rc);
2045	return rc;
2046	}
2047
2048
2049	/**
2050	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2051	* the host-state area in the VMCS.
2052	*
2053	* @returns VBox status code.
2054	* @param pVM Pointer to the VM.
2055	* @param pVCpu Pointer to the VMCPU.
2056	*/
2057	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2058	{
2059	int rc = VERR_INTERNAL_ERROR_5;
2060	RTSEL uSelCS = 0;
2061	RTSEL uSelSS = 0;
2062	RTSEL uSelDS = 0;
2063	RTSEL uSelES = 0;
2064	RTSEL uSelFS = 0;
2065	RTSEL uSelGS = 0;
2066	RTSEL uSelTR = 0;
2067
2068	/*
2069	* Host Selector registers.
2070	*/
2071	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2072	if (VMX_IS_64BIT_HOST_MODE())
2073	{
2074	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2075	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2076	}
2077	else
2078	{
2079	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2080	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2081	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2082	}
2083	#else
2084	uSelCS = ASMGetCS();
2085	uSelSS = ASMGetSS();
2086	#endif
2087
2088	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2089	/** @todo Verify if we have any platform that actually run with DS or ES with
2090	* RPL != 0 in kernel space. */
2091	uSelDS = 0;
2092	uSelES = 0;
2093	uSelFS = 0;
2094	uSelGS = 0;
2095	uSelTR = ASMGetTR();
2096
2097	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2098	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2099	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2100	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2101	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2102	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2103	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2104	Assert(uSelCS != 0);
2105	Assert(uSelTR != 0);
2106
2107	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2108	#if 0
2109	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2110	Assert(uSelSS != 0);
2111	#endif
2112
2113	/* Write these host selector fields into the host-state area in the VMCS. */
2114	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2115	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2116	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2117	#if 0
2118	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2119	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2120	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2121	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2122	#endif
2123	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2124	AssertRCReturn(rc, rc);
2125
2126	/*
2127	* Host GDTR and IDTR.
2128	*/
2129	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2130	* be safe to -not- save and restore GDTR and IDTR in the assembly
2131	* code and just do it here and don't care if the limits are zapped on
2132	* VM-exit. */
2133	RTGDTR Gdtr;
2134	RT_ZERO(Gdtr);
2135	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2136	if (VMX_IS_64BIT_HOST_MODE())
2137	{
2138	X86XDTR64 Gtr64;
2139	X86XDTR64 Idtr64;
2140	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2141	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2142	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2143	Gdtr.cbGdt = Gdtr64.cb;
2144	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2145	}
2146	else
2147	#endif
2148	{
2149	RTIDTR Idtr;
2150	ASMGetGDTR(&Gdtr);
2151	ASMGetIDTR(&Idtr);
2152	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2153	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2154	}
2155	AssertRCReturn(rc, rc);
2156
2157	/*
2158	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2159	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2160	*/
2161	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2162	{
2163	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2164	return VERR_VMX_INVALID_HOST_STATE;
2165	}
2166
2167	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2168	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2169	if (VMX_IS_64BIT_HOST_MODE())
2170	{
2171	/* We need the 64-bit TR base for hybrid darwin. */
2172	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2173	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2174	}
2175	else
2176	#endif
2177	{
2178	uintptr_t uTRBase;
2179	#if HC_ARCH_BITS == 64
2180	uTRBase = X86DESC64_BASE(pDesc);
2181	#else
2182	uTRBase = X86DESC_BASE(pDesc);
2183	#endif
2184	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2185	}
2186	AssertRCReturn(rc, rc);
2187
2188	/*
2189	* Host FS base and GS base.
2190	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2191	* would take care of the bases. In 64-bit, the MSRs come into play.
2192	*/
2193	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2194	if (VMX_IS_64BIT_HOST_MODE())
2195	{
2196	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2197	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2198	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2199	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2200	AssertRCReturn(rc, rc);
2201	}
2202	#endif
2203	return rc;
2204	}
2205
2206
2207	/**
2208	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2209	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2210	* the host after every successful VM exit.
2211	*
2212	* @returns VBox status code.
2213	* @param pVM Pointer to the VM.
2214	* @param pVCpu Pointer to the VMCPU.
2215	*/
2216	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2217	{
2218	AssertPtr(pVCpu);
2219	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2220
2221	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2222	uint32_t cHostMsrs = 0;
2223	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2224
2225	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2226	{
2227	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2228	pHostMsr->u32Reserved = 0;
2229	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2230	if (CPUMIsGuestInLongMode(pVCpu))
2231	{
2232	/* Must match the EFER value in our 64 bits switcher. */
2233	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2234	}
2235	else
2236	#endif
2237	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2238	pHostMsr++; cHostMsrs++;
2239	}
2240
2241	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2242	if (VMX_IS_64BIT_HOST_MODE())
2243	{
2244	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2245	pHostMsr->u32Reserved = 0;
2246	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2247	pHostMsr++; cHostMsrs++;
2248	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2249	pHostMsr->u32Reserved = 0;
2250	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2251	pHostMsr++; cHostMsrs++;
2252	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2253	pHostMsr->u32Reserved = 0;
2254	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2255	pHostMsr++; cHostMsrs++;
2256	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2257	#if 0
2258	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2259	pMsr->u32Reserved = 0;
2260	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2261	pHostMsr++; cHostMsrs++;
2262	#endif
2263	}
2264	#endif
2265
2266	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2267	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2268	{
2269	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2270	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2271	}
2272
2273	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2274
2275	/*
2276	* Host Sysenter MSRs.
2277	*/
2278	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2279	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2280	if (VMX_IS_64BIT_HOST_MODE())
2281	{
2282	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2283	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2284	}
2285	else
2286	{
2287	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2288	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2289	}
2290	#elif HC_ARCH_BITS == 32
2291	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2292	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2293	#else
2294	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2295	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2296	#endif
2297	AssertRCReturn(rc, rc);
2298
2299	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2300	* hmR0VmxSetupExitCtls() !! */
2301	return rc;
2302	}
2303
2304
2305	/**
2306	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2307	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2308	* controls".
2309	*
2310	* @returns VBox status code.
2311	* @param pVM Pointer to the VM.
2312	* @param pVCpu Pointer to the VMCPU.
2313	* @param pCtx Pointer to the guest-CPU context.
2314	*
2315	* @remarks No-long-jump zone!!!
2316	*/
2317	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2318	{
2319	int rc = VINF_SUCCESS;
2320	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2321	{
2322	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2323	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2324
2325	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2326	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2327
2328	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2329	if (CPUMIsGuestInLongModeEx(pCtx))
2330	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2331	else
2332	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2333
2334	/*
2335	* The following should not be set (since we're not in SMM mode):
2336	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2337	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2338	*/
2339
2340	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2341	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2342	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2343
2344	if ((val & zap) != val)
2345	{
2346	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2347	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2348	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2349	}
2350
2351	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2352	AssertRCReturn(rc, rc);
2353
2354	/* Update VCPU with the currently set VM-exit controls. */
2355	pVCpu->hm.s.vmx.u32EntryCtls = val;
2356	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2357	}
2358	return rc;
2359	}
2360
2361
2362	/**
2363	* Sets up the VM-exit controls in the VMCS.
2364	*
2365	* @returns VBox status code.
2366	* @param pVM Pointer to the VM.
2367	* @param pVCpu Pointer to the VMCPU.
2368	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2369	* out-of-sync. Make sure to update the required fields
2370	* before using them.
2371	*
2372	* @remarks requires EFER.
2373	*/
2374	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2375	{
2376	int rc = VINF_SUCCESS;
2377	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2378	{
2379	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2380	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2381
2382	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2383	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2384
2385	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2386	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2387	if (VMX_IS_64BIT_HOST_MODE())
2388	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2389	else
2390	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2391	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2392	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2393	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2394	else
2395	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2396	#endif
2397
2398	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2399	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2400
2401	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2402	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2403	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2404	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2405	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2406
2407	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2408	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2409
2410	if ((val & zap) != val)
2411	{
2412	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2413	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2414	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2415	}
2416
2417	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2418	AssertRCReturn(rc, rc);
2419
2420	/* Update VCPU with the currently set VM-exit controls. */
2421	pVCpu->hm.s.vmx.u32ExitCtls = val;
2422	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2423	}
2424	return rc;
2425	}
2426
2427
2428	/**
2429	* Loads the guest APIC and related state.
2430	*
2431	* @returns VBox status code.
2432	* @param pVM Pointer to the VM.
2433	* @param pVCpu Pointer to the VMCPU.
2434	* @param pCtx Pointer to the guest-CPU context.
2435	*/
2436	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2437	{
2438	int rc = VINF_SUCCESS;
2439	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2440	{
2441	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2442	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2443	{
2444	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2445
2446	bool fPendingIntr = false;
2447	uint8_t u8GuestTpr = 0;
2448	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2449	AssertRCReturn(rc, rc);
2450
2451	/*
2452	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2453	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2454	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2455	* the interrupt when we VM-exit for other reasons.
2456	*/
2457	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2458	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2459	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2460	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2461
2462	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2463	AssertRCReturn(rc, rc);
2464
2465	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2466	if (pVM->hm.s.fTPRPatchingActive)
2467	{
2468	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2469	pCtx->msrLSTAR = u8GuestTpr;
2470	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2471	{
2472	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2473	if (fPendingIntr)
2474	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2475	else
2476	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2477	}
2478	}
2479	}
2480
2481	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2482	}
2483	return rc;
2484	}
2485
2486
2487	/**
2488	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2489	*
2490	* @returns
2491	* @param pVCpu Pointer to the VMCPU.
2492	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2493	* out-of-sync. Make sure to update the required fields
2494	* before using them.
2495	*
2496	* @remarks No-long-jump zone!!!
2497	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2498	*/
2499	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2500	{
2501	/*
2502	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2503	* inhibit interrupts or clear any existing interrupt-inhibition.
2504	*/
2505	uint32_t uIntrState = 0;
2506	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2507	{
2508	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2509	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS))
2510	== (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2511	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2512	{
2513	/*
2514	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2515	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2516	*/
2517	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2518	}
2519	else if (pMixedCtx->eflags.Bits.u1IF)
2520	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2521	else
2522	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2523	}
2524	return uIntrState;
2525	}
2526
2527
2528	/**
2529	* Loads the guest's interruptibility-state into the guest-state area in the
2530	* VMCS.
2531	*
2532	* @returns VBox status code.
2533	* @param pVCpu Pointer to the VMCPU.
2534	* @param uIntrState The interruptibility-state to set.
2535	*/
2536	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2537	{
2538	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2539	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2540	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2541	AssertRCReturn(rc, rc);
2542	return rc;
2543	}
2544
2545
2546	/**
2547	* Loads the guest's RIP into the guest-state area in the VMCS.
2548	*
2549	* @returns VBox status code.
2550	* @param pVM Pointer to the VM.
2551	* @param pVCpu Pointer to the VMCPU.
2552	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2553	* out-of-sync. Make sure to update the required fields
2554	* before using them.
2555	*
2556	* @remarks No-long-jump zone!!!
2557	*/
2558	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2559	{
2560	int rc = VINF_SUCCESS;
2561	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2562	{
2563	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2564	AssertRCReturn(rc, rc);
2565	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2566	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2567	}
2568	return rc;
2569	}
2570
2571
2572	/**
2573	* Loads the guest's RSP into the guest-state area in the VMCS.
2574	*
2575	* @returns VBox status code.
2576	* @param pVM Pointer to the VM.
2577	* @param pVCpu Pointer to the VMCPU.
2578	* @param pCtx Pointer to the guest-CPU context.
2579	*
2580	* @remarks No-long-jump zone!!!
2581	*/
2582	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2583	{
2584	int rc = VINF_SUCCESS;
2585	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2586	{
2587	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2588	AssertRCReturn(rc, rc);
2589	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2590	}
2591	return rc;
2592	}
2593
2594
2595	/**
2596	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2597	*
2598	* @returns VBox status code.
2599	* @param pVM Pointer to the VM.
2600	* @param pVCpu Pointer to the VMCPU.
2601	* @param pCtx Pointer to the guest-CPU context.
2602	*
2603	* @remarks No-long-jump zone!!!
2604	*/
2605	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2606	{
2607	int rc = VINF_SUCCESS;
2608	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2609	{
2610	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2611	Let us assert it as such and use native-width VMWRITE. */
2612	X86RFLAGS uRFlags = pCtx->rflags;
2613	Assert(uRFlags.u64 >> 32 == 0);
2614	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2615	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2616
2617	/*
2618	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2619	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2620	*/
2621	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2622	{
2623	Assert(pVM->hm.s.vmx.pRealModeTSS);
2624	Assert(PDMVmmDevHeapIsEnabled(pVM));
2625	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2626	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2627	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2628	}
2629
2630	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2631	AssertRCReturn(rc, rc);
2632
2633	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2634	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2635	}
2636	return rc;
2637	}
2638
2639
2640	/**
2641	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2642	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2643	* assembly code.
2644	*
2645	* @returns VBox status code.
2646	* @param pVM Pointer to the VM.
2647	* @param pVCpu Pointer to the VMCPU.
2648	* @param pCtx Pointer to the guest-CPU context.
2649	*
2650	* @remarks No-long-jump zone!!!
2651	*/
2652	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2653	{
2654	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2655	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2656	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2657	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2658	return rc;
2659	}
2660
2661
2662	/**
2663	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2664	* in the VMCS.
2665	*
2666	* @returns VBox status code.
2667	* @param pVM Pointer to the VM.
2668	* @param pVCpu Pointer to the VMCPU.
2669	* @param pCtx Pointer to the guest-CPU context.
2670	*
2671	* @remarks No-long-jump zone!!!
2672	*/
2673	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2674	{
2675	int rc = VINF_SUCCESS;
2676
2677	/*
2678	* Guest CR0.
2679	* Guest FPU.
2680	*/
2681	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2682	{
2683	uint64_t u64GuestCR0 = pCtx->cr0;
2684
2685	/* The guest's view (read access) of its CR0 is unblemished. */
2686	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2687	AssertRCReturn(rc, rc);
2688	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2689
2690	/* Setup VT-x's view of the guest CR0. */
2691	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2692	if (pVM->hm.s.fNestedPaging)
2693	{
2694	if (CPUMIsGuestPagingEnabledEx(pCtx))
2695	{
2696	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2697	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2698	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2699	}
2700	else
2701	{
2702	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2703	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2704	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2705	}
2706
2707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2708	AssertRCReturn(rc, rc);
2709	}
2710	else
2711	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2712
2713	/*
2714	* Guest FPU bits.
2715	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2716	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2717	*/
2718	u64GuestCR0 \|= X86_CR0_NE;
2719	bool fInterceptNM = false;
2720	if (CPUMIsGuestFPUStateActive(pVCpu))
2721	{
2722	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2723	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2724	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2725	}
2726	else
2727	{
2728	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2729	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2730	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2731	}
2732
2733	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2734	bool fInterceptMF = false;
2735	if (!(pCtx->cr0 & X86_CR0_NE))
2736	fInterceptMF = true;
2737
2738	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2739	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2740	{
2741	Assert(PDMVmmDevHeapIsEnabled(pVM));
2742	Assert(pVM->hm.s.vmx.pRealModeTSS);
2743	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2744	fInterceptNM = true;
2745	fInterceptMF = true;
2746	}
2747	else
2748	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2749
2750	if (fInterceptNM)
2751	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2752	else
2753	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2754
2755	if (fInterceptMF)
2756	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2757	else
2758	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2759
2760	/* Additional intercepts for debugging, define these yourself explicitly. */
2761	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2762	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2763	\| RT_BIT(X86_XCPT_DB)
2764	\| RT_BIT(X86_XCPT_DE)
2765	\| RT_BIT(X86_XCPT_NM)
2766	\| RT_BIT(X86_XCPT_UD)
2767	\| RT_BIT(X86_XCPT_NP)
2768	\| RT_BIT(X86_XCPT_SS)
2769	\| RT_BIT(X86_XCPT_GP)
2770	\| RT_BIT(X86_XCPT_PF)
2771	\| RT_BIT(X86_XCPT_MF);
2772	#elif defined(VBOX_ALWAYS_TRAP_PF)
2773	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2774	#endif
2775
2776	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2777
2778	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2779	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2780	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2781	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2782	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2783	else
2784	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2785
2786	u64GuestCR0 \|= uSetCR0;
2787	u64GuestCR0 &= uZapCR0;
2788	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2789
2790	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2791	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2792	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2793	Log(("Load: VMX_VMCS_GUEST_CR0=%RGv (uSetCR0=%#RGv uZapCR0=%#RGv)\n", u64GuestCR0, uSetCR0, uZapCR0));
2794
2795	/*
2796	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2797	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2798	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2799	*/
2800	uint64_t u64CR0Mask = 0;
2801	u64CR0Mask = X86_CR0_PE
2802	\| X86_CR0_NE
2803	\| X86_CR0_WP
2804	\| X86_CR0_PG
2805	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2806	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2807	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2808	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2809	u64CR0Mask &= ~X86_CR0_PE;
2810	/* Enable this later. */
2811	/* if (pVM->hm.s.fNestedPaging)
2812	u64CR0Mask &= ~X86_CR0_WP; */
2813
2814	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2815	if (fInterceptNM)
2816	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2817	else
2818	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2819
2820	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2821	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2822	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2823	AssertRCReturn(rc, rc);
2824
2825	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2826	}
2827
2828	/*
2829	* Guest CR2.
2830	* It's always loaded in the assembler code. Nothing to do here.
2831	*/
2832
2833	/*
2834	* Guest CR3.
2835	*/
2836	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2837	{
2838	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2839	if (pVM->hm.s.fNestedPaging)
2840	{
2841	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2842
2843	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2844	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2845	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2846	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2847
2848	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2849	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2850	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2851
2852	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2853	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2854	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2855	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2856
2857	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2858	AssertRCReturn(rc, rc);
2859	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2860
2861	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2862	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2863	{
2864	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2865	if (CPUMIsGuestInPAEModeEx(pCtx))
2866	{
2867	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2868	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2869	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2870	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2871	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2872	AssertRCReturn(rc, rc);
2873	}
2874
2875	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2876	have Unrestricted Execution to handle the guest when it's not using paging. */
2877	GCPhysGuestCR3 = pCtx->cr3;
2878	}
2879	else
2880	{
2881	/*
2882	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2883	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2884	* EPT takes care of translating it to host-physical addresses.
2885	*/
2886	RTGCPHYS GCPhys;
2887	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2888	Assert(PDMVmmDevHeapIsEnabled(pVM));
2889
2890	/* We obtain it here every time as the guest could have relocated this PCI region. */
2891	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2892	AssertRCReturn(rc, rc);
2893
2894	GCPhysGuestCR3 = GCPhys;
2895	}
2896	}
2897	else
2898	{
2899	/* Non-nested paging case, just use the hypervisor's CR3. */
2900	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2901	}
2902
2903	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2904	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2905	AssertRCReturn(rc, rc);
2906
2907	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2908	}
2909
2910	/*
2911	* Guest CR4.
2912	*/
2913	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2914	{
2915	uint32_t u64GuestCR4 = pCtx->cr4;
2916	Assert(!(pCtx->cr4 >> 32));
2917
2918	/* The guest's view of its CR4 is unblemished. */
2919	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2920	AssertRCReturn(rc, rc);
2921	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2922
2923	/* Setup VT-x's view of the guest CR4. */
2924	/*
2925	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2926	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2927	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2928	*/
2929	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2930	{
2931	Assert(pVM->hm.s.vmx.pRealModeTSS);
2932	Assert(PDMVmmDevHeapIsEnabled(pVM));
2933	u64GuestCR4 &= ~X86_CR4_VME;
2934	}
2935
2936	if (pVM->hm.s.fNestedPaging)
2937	{
2938	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2939	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2940	{
2941	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2942	u64GuestCR4 \|= X86_CR4_PSE;
2943	/* Our identity mapping is a 32 bits page directory. */
2944	u64GuestCR4 &= ~X86_CR4_PAE;
2945	}
2946	/* else use guest CR4.*/
2947	}
2948	else
2949	{
2950	/*
2951	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2952	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2953	*/
2954	switch (pVCpu->hm.s.enmShadowMode)
2955	{
2956	case PGMMODE_REAL: /* Real-mode. */
2957	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2958	case PGMMODE_32_BIT: /* 32-bit paging. */
2959	{
2960	u64GuestCR4 &= ~X86_CR4_PAE;
2961	break;
2962	}
2963
2964	case PGMMODE_PAE: /* PAE paging. */
2965	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2966	{
2967	u64GuestCR4 \|= X86_CR4_PAE;
2968	break;
2969	}
2970
2971	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2972	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2973	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2974	break;
2975	#endif
2976	default:
2977	AssertFailed();
2978	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2979	}
2980	}
2981
2982	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2983	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2984	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2985	u64GuestCR4 \|= uSetCR4;
2986	u64GuestCR4 &= uZapCR4;
2987
2988	/* Write VT-x's view of the guest CR4 into the VMCS. */
2989	Log(("Load: VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2990	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2991
2992	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2993	uint64_t u64CR4Mask = 0;
2994	u64CR4Mask = X86_CR4_VME
2995	\| X86_CR4_PAE
2996	\| X86_CR4_PGE
2997	\| X86_CR4_PSE
2998	\| X86_CR4_VMXE;
2999	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3000	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3001	AssertRCReturn(rc, rc);
3002
3003	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3004	}
3005	return rc;
3006	}
3007
3008
3009	/**
3010	* Loads the guest debug registers into the guest-state area in the VMCS.
3011	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3012	*
3013	* @returns VBox status code.
3014	* @param pVM Pointer to the VM.
3015	* @param pVCpu Pointer to the VMCPU.
3016	* @param pCtx Pointer to the guest-CPU context.
3017	*
3018	* @remarks No-long-jump zone!!!
3019	*/
3020	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3021	{
3022	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3023	return VINF_SUCCESS;
3024
3025	#ifdef VBOX_STRICT
3026	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3027	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3028	{
3029	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3030	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3031	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3032	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3033	}
3034	#endif
3035
3036	int rc = VERR_INTERNAL_ERROR_5;
3037	bool fInterceptDB = false;
3038	bool fInterceptMovDRx = false;
3039	if (DBGFIsStepping(pVCpu))
3040	{
3041	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
3042	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3043	{
3044	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3045	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3046	AssertRCReturn(rc, rc);
3047	Assert(fInterceptDB == false);
3048	}
3049	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3050	{
3051	Assert(fInterceptDB == false);
3052	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3053	* just continuing? */
3054	}
3055	else
3056	fInterceptDB = true;
3057	}
3058	else
3059	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3060
3061
3062	/*
3063	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3064	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3065	* The same for the hypervisor DRx registers, priority is for the guest here.
3066	*/
3067	if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3068	{
3069	if (!CPUMIsGuestDebugStateActive(pVCpu))
3070	{
3071	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3072	AssertRC(rc);
3073	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3074	}
3075	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3076	Assert(fInterceptMovDRx == false);
3077	}
3078	else if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3079	{
3080	if (!CPUMIsHyperDebugStateActive(pVCpu))
3081	{
3082	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3083	AssertRC(rc);
3084	}
3085	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3086	fInterceptMovDRx = true;
3087	}
3088	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3089	{
3090	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug state isn't active. */
3091	fInterceptMovDRx = true;
3092	}
3093
3094	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3095	if (fInterceptDB)
3096	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3097	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3098	{
3099	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3100	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3101	#endif
3102	}
3103
3104	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3105	if (fInterceptMovDRx)
3106	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3107	else
3108	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3109
3110	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3111	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3112
3113	/* The guest's view of its DR7 is unblemished. */
3114	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3115
3116	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3117	return rc;
3118	}
3119
3120
3121	#ifdef VBOX_STRICT
3122	/**
3123	* Strict function to validate segment registers.
3124	*/
3125	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3126	{
3127	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3128	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3129	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3130	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3131	{
3132	/* Protected mode checks */
3133	/* CS */
3134	Assert(pCtx->cs.Attr.n.u1Present);
3135	Assert(!(pCtx->cs.Attr.u & 0xf00));
3136	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3137	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3138	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3139	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3140	\|\| (pCtx->cs.Attr.n.u1Granularity));
3141	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3142	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3143	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3144	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3145	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3146	else
3147	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3148	/* SS */
3149	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3150	{
3151	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3152	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3153	Assert(pCtx->ss.Attr.n.u1Present);
3154	Assert(!(pCtx->ss.Attr.u & 0xf00));
3155	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3156	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3157	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3158	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3159	\|\| (pCtx->ss.Attr.n.u1Granularity));
3160	}
3161	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3162	/* CR0 might not be up-to-date here always, hence disabled. */
3163	#if 0
3164	if (!pCtx->cr0 & X86_CR0_PE)
3165	Assert(!pCtx->ss.Attr.n.u2Dpl);
3166	#endif
3167	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3168	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3169	{
3170	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3171	Assert(pCtx->ds.Attr.n.u1Present);
3172	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3173	Assert(!(pCtx->ds.Attr.u & 0xf00));
3174	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3175	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3176	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3177	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3178	\|\| (pCtx->ds.Attr.n.u1Granularity));
3179	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3180	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3181	}
3182	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3183	{
3184	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3185	Assert(pCtx->es.Attr.n.u1Present);
3186	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3187	Assert(!(pCtx->es.Attr.u & 0xf00));
3188	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3189	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3190	\|\| !(pCtx->es.Attr.n.u1Granularity));
3191	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3192	\|\| (pCtx->es.Attr.n.u1Granularity));
3193	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3194	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3195	}
3196	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3197	{
3198	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3199	Assert(pCtx->fs.Attr.n.u1Present);
3200	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3201	Assert(!(pCtx->fs.Attr.u & 0xf00));
3202	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3203	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3204	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3205	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3206	\|\| (pCtx->fs.Attr.n.u1Granularity));
3207	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3208	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3209	}
3210	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3211	{
3212	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3213	Assert(pCtx->gs.Attr.n.u1Present);
3214	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3215	Assert(!(pCtx->gs.Attr.u & 0xf00));
3216	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3217	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3218	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3219	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3220	\|\| (pCtx->gs.Attr.n.u1Granularity));
3221	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3222	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3223	}
3224	/* 64-bit capable CPUs. */
3225	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3226	Assert(!(pCtx->cs.u64Base >> 32));
3227	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3228	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3229	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3230	# endif
3231	}
3232	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3233	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3234	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3235	{
3236	/* Real and v86 mode checks. */
3237	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3238	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3239	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3240	{
3241	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3242	}
3243	else
3244	{
3245	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3246	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3247	}
3248
3249	/* CS */
3250	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3251	Assert(pCtx->cs.u32Limit == 0xffff);
3252	Assert(u32CSAttr == 0xf3);
3253	/* SS */
3254	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3255	Assert(pCtx->ss.u32Limit == 0xffff);
3256	Assert(u32SSAttr == 0xf3);
3257	/* DS */
3258	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3259	Assert(pCtx->ds.u32Limit == 0xffff);
3260	Assert(u32DSAttr == 0xf3);
3261	/* ES */
3262	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3263	Assert(pCtx->es.u32Limit == 0xffff);
3264	Assert(u32ESAttr == 0xf3);
3265	/* FS */
3266	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3267	Assert(pCtx->fs.u32Limit == 0xffff);
3268	Assert(u32FSAttr == 0xf3);
3269	/* GS */
3270	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3271	Assert(pCtx->gs.u32Limit == 0xffff);
3272	Assert(u32GSAttr == 0xf3);
3273	/* 64-bit capable CPUs. */
3274	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3275	Assert(!(pCtx->cs.u64Base >> 32));
3276	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3277	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3278	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3279	# endif
3280	}
3281	}
3282	#endif /* VBOX_STRICT */
3283
3284
3285	/**
3286	* Writes a guest segment register into the guest-state area in the VMCS.
3287	*
3288	* @returns VBox status code.
3289	* @param pVM Pointer to the VM.
3290	* @param pVCpu Pointer to the VMCPU.
3291	* @param idxSel Index of the selector in the VMCS.
3292	* @param idxLimit Index of the segment limit in the VMCS.
3293	* @param idxBase Index of the segment base in the VMCS.
3294	* @param idxAccess Index of the access rights of the segment in the VMCS.
3295	* @param pSelReg Pointer to the segment selector.
3296	* @param pCtx Pointer to the guest-CPU context.
3297	*
3298	* @remarks No-long-jump zone!!!
3299	*/
3300	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3301	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3302	{
3303	int rc;
3304	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3305	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3306	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3307	AssertRCReturn(rc, rc);
3308
3309	uint32_t u32Access = pSelReg->Attr.u;
3310	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3311	{
3312	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3313	u32Access = 0xf3;
3314	Assert(pVM->hm.s.vmx.pRealModeTSS);
3315	Assert(PDMVmmDevHeapIsEnabled(pVM));
3316	}
3317	else
3318	{
3319	/*
3320	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3321	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3322	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3323	* loaded in protected-mode have their attribute as 0.
3324	*/
3325	if (!u32Access)
3326	u32Access = VMX_SEL_UNUSABLE;
3327	}
3328
3329	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3330	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3331	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3332
3333	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3334	AssertRCReturn(rc, rc);
3335	return rc;
3336	}
3337
3338
3339	/**
3340	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3341	* into the guest-state area in the VMCS.
3342	*
3343	* @returns VBox status code.
3344	* @param pVM Pointer to the VM.
3345	* @param pVCPU Pointer to the VMCPU.
3346	* @param pCtx Pointer to the guest-CPU context.
3347	*
3348	* @remarks No-long-jump zone!!!
3349	*/
3350	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3351	{
3352	int rc = VERR_INTERNAL_ERROR_5;
3353
3354	/*
3355	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3356	*/
3357	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3358	{
3359	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3360	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3361	{
3362	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3363	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3364	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3365	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3366	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3367	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3368	}
3369
3370	#ifdef VBOX_WITH_REM
3371	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3372	{
3373	Assert(pVM->hm.s.vmx.pRealModeTSS);
3374	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3375	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3376	{
3377	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3378	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3379	&& enmGuestMode >= PGMMODE_PROTECTED)
3380	{
3381	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3382	in real-mode (e.g. OpenBSD 4.0) */
3383	REMFlushTBs(pVM);
3384	Log(("Load: Switch to protected mode detected!\n"));
3385	}
3386	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3387	}
3388	}
3389	#endif
3390	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3391	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3392	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3393	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3394	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3395	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3396	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3397	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3398	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3399	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3400	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3401	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3402	AssertRCReturn(rc, rc);
3403
3404	#ifdef VBOX_STRICT
3405	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3406	#endif
3407	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3408	}
3409
3410	/*
3411	* Guest TR.
3412	*/
3413	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3414	{
3415	/*
3416	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3417	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3418	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3419	*/
3420	uint16_t u16Sel = 0;
3421	uint32_t u32Limit = 0;
3422	uint64_t u64Base = 0;
3423	uint32_t u32AccessRights = 0;
3424
3425	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3426	{
3427	u16Sel = pCtx->tr.Sel;
3428	u32Limit = pCtx->tr.u32Limit;
3429	u64Base = pCtx->tr.u64Base;
3430	u32AccessRights = pCtx->tr.Attr.u;
3431	}
3432	else
3433	{
3434	Assert(pVM->hm.s.vmx.pRealModeTSS);
3435	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3436
3437	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3438	RTGCPHYS GCPhys;
3439	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3440	AssertRCReturn(rc, rc);
3441
3442	X86DESCATTR DescAttr;
3443	DescAttr.u = 0;
3444	DescAttr.n.u1Present = 1;
3445	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3446
3447	u16Sel = 0;
3448	u32Limit = HM_VTX_TSS_SIZE;
3449	u64Base = GCPhys; /* in real-mode phys = virt. */
3450	u32AccessRights = DescAttr.u;
3451	}
3452
3453	/* Validate. */
3454	Assert(!(u16Sel & RT_BIT(2)));
3455	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3456	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3457	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3458	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3459	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3460	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3461	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3462	Assert( (u32Limit & 0xfff) == 0xfff
3463	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3464	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3465	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3466
3467	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3468	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3469	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3470	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3471	AssertRCReturn(rc, rc);
3472
3473	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3474	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3475	}
3476
3477	/*
3478	* Guest GDTR.
3479	*/
3480	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3481	{
3482	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3483	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3484	AssertRCReturn(rc, rc);
3485
3486	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3487	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3488	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3489	}
3490
3491	/*
3492	* Guest LDTR.
3493	*/
3494	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3495	{
3496	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3497	uint32_t u32Access = 0;
3498	if (!pCtx->ldtr.Attr.u)
3499	u32Access = VMX_SEL_UNUSABLE;
3500	else
3501	u32Access = pCtx->ldtr.Attr.u;
3502
3503	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3504	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3505	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3506	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3507	AssertRCReturn(rc, rc);
3508
3509	/* Validate. */
3510	if (!(u32Access & VMX_SEL_UNUSABLE))
3511	{
3512	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3513	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3514	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3515	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3516	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3517	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3518	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3519	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3520	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3521	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3522	}
3523
3524	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3525	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3526	}
3527
3528	/*
3529	* Guest IDTR.
3530	*/
3531	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3532	{
3533	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3534	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3535	AssertRCReturn(rc, rc);
3536
3537	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3538	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3539	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3540	}
3541
3542	return VINF_SUCCESS;
3543	}
3544
3545
3546	/**
3547	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3548	* areas. These MSRs will automatically be loaded to the host CPU on every
3549	* successful VM entry and stored from the host CPU on every successful VM exit.
3550	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3551	*
3552	* @returns VBox status code.
3553	* @param pVM Pointer to the VM.
3554	* @param pVCpu Pointer to the VMCPU.
3555	* @param pCtx Pointer to the guest-CPU context.
3556	*
3557	* @remarks No-long-jump zone!!!
3558	*/
3559	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3560	{
3561	AssertPtr(pVCpu);
3562	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3563
3564	/*
3565	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3566	*/
3567	int rc = VINF_SUCCESS;
3568	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3569	{
3570	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3571	uint32_t cGuestMsrs = 0;
3572
3573	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3574	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3575	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3576	if (fSupportsNX \|\| fSupportsLongMode)
3577	{
3578	/** @todo support save IA32_EFER, i.e.
3579	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3580	* guest EFER need not be part of the VM-entry MSR-load area. */
3581	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3582	pGuestMsr->u32Reserved = 0;
3583	pGuestMsr->u64Value = pCtx->msrEFER;
3584	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3585	if (!CPUMIsGuestInLongModeEx(pCtx))
3586	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3587	pGuestMsr++; cGuestMsrs++;
3588	if (fSupportsLongMode)
3589	{
3590	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3591	pGuestMsr->u32Reserved = 0;
3592	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3593	pGuestMsr++; cGuestMsrs++;
3594	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3595	pGuestMsr->u32Reserved = 0;
3596	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3597	pGuestMsr++; cGuestMsrs++;
3598	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3599	pGuestMsr->u32Reserved = 0;
3600	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3601	pGuestMsr++; cGuestMsrs++;
3602	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3603	#if 0
3604	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3605	pGuestMsr->u32Reserved = 0;
3606	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3607	pGuestMsr++; cGuestMsrs++;
3608	#endif
3609	}
3610	}
3611
3612	/*
3613	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3614	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3615	*/
3616	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3617	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3618	{
3619	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3620	pGuestMsr->u32Reserved = 0;
3621	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3622	AssertRCReturn(rc, rc);
3623	pGuestMsr++; cGuestMsrs++;
3624	}
3625
3626	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3627	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3628	{
3629	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3630	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3631	}
3632
3633	/* Update the VCPU's copy of the guest MSR count. */
3634	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3635	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3636	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3637	AssertRCReturn(rc, rc);
3638
3639	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3640	}
3641
3642	/*
3643	* Guest Sysenter MSRs.
3644	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3645	* VM exits on WRMSRs for these MSRs.
3646	*/
3647	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3648	{
3649	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3650	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3651	}
3652	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3653	{
3654	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3656	}
3657	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3658	{
3659	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3661	}
3662	AssertRCReturn(rc, rc);
3663
3664	return rc;
3665	}
3666
3667
3668	/**
3669	* Loads the guest activity state into the guest-state area in the VMCS.
3670	*
3671	* @returns VBox status code.
3672	* @param pVM Pointer to the VM.
3673	* @param pVCpu Pointer to the VMCPU.
3674	* @param pCtx Pointer to the guest-CPU context.
3675	*
3676	* @remarks No-long-jump zone!!!
3677	*/
3678	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3679	{
3680	/** @todo See if we can make use of other states, e.g.
3681	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3682	int rc = VINF_SUCCESS;
3683	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3684	{
3685	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3686	AssertRCReturn(rc, rc);
3687	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3688	}
3689	return rc;
3690	}
3691
3692
3693	/**
3694	* Sets up the appropriate function to run guest code.
3695	*
3696	* @returns VBox status code.
3697	* @param pVM Pointer to the VM.
3698	* @param pVCpu Pointer to the VMCPU.
3699	* @param pCtx Pointer to the guest-CPU context.
3700	*
3701	* @remarks No-long-jump zone!!!
3702	*/
3703	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3704	{
3705	if (CPUMIsGuestInLongModeEx(pCtx))
3706	{
3707	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3708	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3709	#endif
3710	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3711	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3712	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3713	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3714	#else
3715	/* 64-bit host or hybrid host. */
3716	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3717	#endif
3718	}
3719	else
3720	{
3721	/* Guest is not in long mode, use the 32-bit handler. */
3722	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3723	}
3724	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3725	return VINF_SUCCESS;
3726	}
3727
3728
3729	/**
3730	* Wrapper for running the guest code in VT-x.
3731	*
3732	* @returns VBox strict status code.
3733	* @param pVM Pointer to the VM.
3734	* @param pVCpu Pointer to the VMCPU.
3735	* @param pCtx Pointer to the guest-CPU context.
3736	*
3737	* @remarks No-long-jump zone!!!
3738	*/
3739	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3740	{
3741	/*
3742	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3743	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3744	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3745	*/
3746	#ifdef VBOX_WITH_KERNEL_USING_XMM
3747	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3748	#else
3749	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3750	#endif
3751	}
3752
3753
3754	/**
3755	* Report world-switch error and dump some useful debug info.
3756	*
3757	* @param pVM Pointer to the VM.
3758	* @param pVCpu Pointer to the VMCPU.
3759	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3760	* @param pCtx Pointer to the guest-CPU context.
3761	* @param pVmxTransient Pointer to the VMX transient structure (only
3762	* exitReason updated).
3763	*/
3764	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3765	{
3766	Assert(pVM);
3767	Assert(pVCpu);
3768	Assert(pCtx);
3769	Assert(pVmxTransient);
3770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3771
3772	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3773	switch (rcVMRun)
3774	{
3775	case VERR_VMX_INVALID_VMXON_PTR:
3776	AssertFailed();
3777	break;
3778	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3779	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3780	{
3781	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3782	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3783	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3784	AssertRC(rc);
3785
3786	#ifdef VBOX_STRICT
3787	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3788	pVmxTransient->uExitReason));
3789	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3790	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3791	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3792	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3793	else
3794	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3795
3796	/* VMX control bits. */
3797	uint32_t u32Val;
3798	uint64_t u64Val;
3799	RTHCUINTREG uHCReg;
3800	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3801	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3802	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3803	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3804	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3805	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3806	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3807	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3808	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3809	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3810	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3811	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3812	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3813	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3814	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3815	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3816	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3817	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3818	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3819	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3820	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3821	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3822	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3823	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3824	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3826	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3827	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3828	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3829	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3830	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3831	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3832	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3833	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3834	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3835	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3836	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3837	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3838	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3839	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3840	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3841	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3842
3843	/* Guest bits. */
3844	RTGCUINTREG uGCReg;
3845	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3846	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3847	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3848	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3849	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3850	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3851	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3852	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3853
3854	/* Host bits. */
3855	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3856	Log(("Host CR0 %#RHr\n", uHCReg));
3857	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3858	Log(("Host CR3 %#RHr\n", uHCReg));
3859	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3860	Log(("Host CR4 %#RHr\n", uHCReg));
3861
3862	RTGDTR HostGdtr;
3863	PCX86DESCHC pDesc;
3864	ASMGetGDTR(&HostGdtr);
3865	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3866	Log(("Host CS %#08x\n", u32Val));
3867	if (u32Val < HostGdtr.cbGdt)
3868	{
3869	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3870	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3871	}
3872
3873	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3874	Log(("Host DS %#08x\n", u32Val));
3875	if (u32Val < HostGdtr.cbGdt)
3876	{
3877	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3878	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3879	}
3880
3881	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3882	Log(("Host ES %#08x\n", u32Val));
3883	if (u32Val < HostGdtr.cbGdt)
3884	{
3885	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3886	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3887	}
3888
3889	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3890	Log(("Host FS %#08x\n", u32Val));
3891	if (u32Val < HostGdtr.cbGdt)
3892	{
3893	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3894	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3895	}
3896
3897	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3898	Log(("Host GS %#08x\n", u32Val));
3899	if (u32Val < HostGdtr.cbGdt)
3900	{
3901	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3902	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3903	}
3904
3905	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3906	Log(("Host SS %#08x\n", u32Val));
3907	if (u32Val < HostGdtr.cbGdt)
3908	{
3909	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3910	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3911	}
3912
3913	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3914	Log(("Host TR %#08x\n", u32Val));
3915	if (u32Val < HostGdtr.cbGdt)
3916	{
3917	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3918	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3919	}
3920
3921	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3922	Log(("Host TR Base %#RHv\n", uHCReg));
3923	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3924	Log(("Host GDTR Base %#RHv\n", uHCReg));
3925	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3926	Log(("Host IDTR Base %#RHv\n", uHCReg));
3927	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3928	Log(("Host SYSENTER CS %#08x\n", u32Val));
3929	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3930	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3931	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3932	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3933	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3934	Log(("Host RSP %#RHv\n", uHCReg));
3935	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3936	Log(("Host RIP %#RHv\n", uHCReg));
3937	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3938	if (VMX_IS_64BIT_HOST_MODE())
3939	{
3940	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3941	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3942	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3943	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3944	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3945	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3946	}
3947	# endif
3948	#endif /* VBOX_STRICT */
3949	break;
3950	}
3951
3952	default:
3953	/* Impossible */
3954	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3955	break;
3956	}
3957	NOREF(pVM);
3958	}
3959
3960
3961	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3962	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3963	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3964	#endif
3965
3966	#ifdef VBOX_STRICT
3967	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3968	{
3969	switch (idxField)
3970	{
3971	case VMX_VMCS_GUEST_RIP:
3972	case VMX_VMCS_GUEST_RSP:
3973	case VMX_VMCS_GUEST_RFLAGS:
3974	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3975	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3976	case VMX_VMCS_GUEST_CR0:
3977	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3978	case VMX_VMCS_GUEST_CR4:
3979	case VMX_VMCS_GUEST_DR7:
3980	case VMX_VMCS32_GUEST_SYSENTER_CS:
3981	case VMX_VMCS_GUEST_SYSENTER_EIP:
3982	case VMX_VMCS_GUEST_SYSENTER_ESP:
3983	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3984	case VMX_VMCS_GUEST_GDTR_BASE:
3985	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3986	case VMX_VMCS_GUEST_IDTR_BASE:
3987	case VMX_VMCS16_GUEST_FIELD_CS:
3988	case VMX_VMCS32_GUEST_CS_LIMIT:
3989	case VMX_VMCS_GUEST_CS_BASE:
3990	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3991	case VMX_VMCS16_GUEST_FIELD_DS:
3992	case VMX_VMCS32_GUEST_DS_LIMIT:
3993	case VMX_VMCS_GUEST_DS_BASE:
3994	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
3995	case VMX_VMCS16_GUEST_FIELD_ES:
3996	case VMX_VMCS32_GUEST_ES_LIMIT:
3997	case VMX_VMCS_GUEST_ES_BASE:
3998	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
3999	case VMX_VMCS16_GUEST_FIELD_FS:
4000	case VMX_VMCS32_GUEST_FS_LIMIT:
4001	case VMX_VMCS_GUEST_FS_BASE:
4002	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4003	case VMX_VMCS16_GUEST_FIELD_GS:
4004	case VMX_VMCS32_GUEST_GS_LIMIT:
4005	case VMX_VMCS_GUEST_GS_BASE:
4006	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4007	case VMX_VMCS16_GUEST_FIELD_SS:
4008	case VMX_VMCS32_GUEST_SS_LIMIT:
4009	case VMX_VMCS_GUEST_SS_BASE:
4010	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4011	case VMX_VMCS16_GUEST_FIELD_LDTR:
4012	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4013	case VMX_VMCS_GUEST_LDTR_BASE:
4014	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4015	case VMX_VMCS16_GUEST_FIELD_TR:
4016	case VMX_VMCS32_GUEST_TR_LIMIT:
4017	case VMX_VMCS_GUEST_TR_BASE:
4018	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4019	case VMX_VMCS32_RO_EXIT_REASON:
4020	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4021	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4022	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4023	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4024	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4025	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4026	case VMX_VMCS32_RO_IDT_INFO:
4027	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4028	case VMX_VMCS_GUEST_CR3:
4029	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4030	return true;
4031	}
4032	return false;
4033	}
4034
4035	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4036	{
4037	switch (idxField)
4038	{
4039	case VMX_VMCS_GUEST_LDTR_BASE:
4040	case VMX_VMCS_GUEST_TR_BASE:
4041	case VMX_VMCS_GUEST_GDTR_BASE:
4042	case VMX_VMCS_GUEST_IDTR_BASE:
4043	case VMX_VMCS_GUEST_SYSENTER_EIP:
4044	case VMX_VMCS_GUEST_SYSENTER_ESP:
4045	case VMX_VMCS_GUEST_CR0:
4046	case VMX_VMCS_GUEST_CR4:
4047	case VMX_VMCS_GUEST_CR3:
4048	case VMX_VMCS_GUEST_DR7:
4049	case VMX_VMCS_GUEST_RIP:
4050	case VMX_VMCS_GUEST_RSP:
4051	case VMX_VMCS_GUEST_CS_BASE:
4052	case VMX_VMCS_GUEST_DS_BASE:
4053	case VMX_VMCS_GUEST_ES_BASE:
4054	case VMX_VMCS_GUEST_FS_BASE:
4055	case VMX_VMCS_GUEST_GS_BASE:
4056	case VMX_VMCS_GUEST_SS_BASE:
4057	return true;
4058	}
4059	return false;
4060	}
4061	#endif /* VBOX_STRICT */
4062
4063	/**
4064	* Executes the specified handler in 64-bit mode.
4065	*
4066	* @returns VBox status code.
4067	* @param pVM Pointer to the VM.
4068	* @param pVCpu Pointer to the VMCPU.
4069	* @param pCtx Pointer to the guest CPU context.
4070	* @param pfnHandler Pointer to the RC handler function.
4071	* @param cbParam Number of parameters.
4072	* @param paParam Array of 32-bit parameters.
4073	*/
4074	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4075	uint32_t *paParam)
4076	{
4077	int rc, rc2;
4078	PHMGLOBLCPUINFO pCpu;
4079	RTHCPHYS HCPhysCpuPage;
4080	RTHCUINTREG uOldEFlags;
4081
4082	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4083	Assert(pfnHandler);
4084	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4085	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4086
4087	#ifdef VBOX_STRICT
4088	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4089	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4090
4091	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4092	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4093	#endif
4094
4095	/* Disable interrupts. */
4096	uOldEFlags = ASMIntDisableFlags();
4097
4098	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4099	RTCPUID idHostCpu = RTMpCpuId();
4100	CPUMR0SetLApic(pVM, idHostCpu);
4101	#endif
4102
4103	pCpu = HMR0GetCurrentCpu();
4104	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4105
4106	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4107	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4108
4109	/* Leave VMX Root Mode. */
4110	VMXDisable();
4111
4112	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4113
4114	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4115	CPUMSetHyperEIP(pVCpu, pfnHandler);
4116	for (int i = (int)cbParam - 1; i >= 0; i--)
4117	CPUMPushHyper(pVCpu, paParam[i]);
4118
4119	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4120
4121	/* Call the switcher. */
4122	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4123	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4124
4125	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4126	/* Make sure the VMX instructions don't cause #UD faults. */
4127	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4128
4129	/* Re-enter VMX Root Mode */
4130	rc2 = VMXEnable(HCPhysCpuPage);
4131	if (RT_FAILURE(rc2))
4132	{
4133	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4134	ASMSetFlags(uOldEFlags);
4135	return rc2;
4136	}
4137
4138	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4139	AssertRC(rc2);
4140	Assert(!(ASMGetFlags() & X86_EFL_IF));
4141	ASMSetFlags(uOldEFlags);
4142	return rc;
4143	}
4144
4145
4146	/**
4147	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4148	* supporting 64-bit guests.
4149	*
4150	* @returns VBox status code.
4151	* @param fResume Whether to VMLAUNCH or VMRESUME.
4152	* @param pCtx Pointer to the guest-CPU context.
4153	* @param pCache Pointer to the VMCS cache.
4154	* @param pVM Pointer to the VM.
4155	* @param pVCpu Pointer to the VMCPU.
4156	*/
4157	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4158	{
4159	uint32_t aParam[6];
4160	PHMGLOBLCPUINFO pCpu = NULL;
4161	RTHCPHYS HCPhysCpuPage = 0;
4162	int rc = VERR_INTERNAL_ERROR_5;
4163
4164	pCpu = HMR0GetCurrentCpu();
4165	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4166
4167	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4168	pCache->uPos = 1;
4169	pCache->interPD = PGMGetInterPaeCR3(pVM);
4170	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4171	#endif
4172
4173	#ifdef VBOX_STRICT
4174	pCache->TestIn.HCPhysCpuPage = 0;
4175	pCache->TestIn.HCPhysVmcs = 0;
4176	pCache->TestIn.pCache = 0;
4177	pCache->TestOut.HCPhysVmcs = 0;
4178	pCache->TestOut.pCache = 0;
4179	pCache->TestOut.pCtx = 0;
4180	pCache->TestOut.eflags = 0;
4181	#endif
4182
4183	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4184	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4185	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4186	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4187	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4188	aParam[5] = 0;
4189
4190	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4191	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4192	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4193	#endif
4194	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4195
4196	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4197	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4198	Assert(pCtx->dr[4] == 10);
4199	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4200	#endif
4201
4202	#ifdef VBOX_STRICT
4203	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4204	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4205	pVCpu->hm.s.vmx.HCPhysVmcs));
4206	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4207	pCache->TestOut.HCPhysVmcs));
4208	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4209	pCache->TestOut.pCache));
4210	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4211	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4212	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4213	pCache->TestOut.pCtx));
4214	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4215	#endif
4216	return rc;
4217	}
4218
4219
4220	/**
4221	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4222	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4223	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4224	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4225	*
4226	* @returns VBox status code.
4227	* @param pVM Pointer to the VM.
4228	* @param pVCpu Pointer to the VMCPU.
4229	*/
4230	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4231	{
4232	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4233	{ \
4234	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4235	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4236	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4237	}
4238
4239	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4240	{ \
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4245	}
4246
4247	AssertPtr(pVM);
4248	AssertPtr(pVCpu);
4249	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4250
4251	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4252	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4253	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4254	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4255	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4256	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4257	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4258	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4259	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4260
4261	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4262	#if 0
4263	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4264	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4265	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4272	#endif
4273
4274	/* 32-bit guest-state fields. */
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4279	/* Unused 32-bit guest-state fields. */
4280	#if 0
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4284	#endif
4285
4286	/* Natural width guest-state fields. */
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4290	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4303	/* Unused natural width guest-state fields. */
4304	#if 0
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4307	#endif
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4310
4311	if (pVM->hm.s.fNestedPaging)
4312	{
4313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4315	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4316	}
4317	else
4318	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4319
4320	#undef VMXLOCAL_INIT_VMCS_SELREG
4321	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4322	return VINF_SUCCESS;
4323	}
4324
4325
4326	/**
4327	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4328	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4329	* darwin, running 64-bit guests).
4330	*
4331	* @returns VBox status code.
4332	* @param pVCpu Pointer to the VMCPU.
4333	* @param idxField The VMCS field encoding.
4334	* @param u64Val 16, 32 or 64 bits value.
4335	*/
4336	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4337	{
4338	int rc;
4339	switch (idxField)
4340	{
4341	/*
4342	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4343	*/
4344	/* 64-bit Control fields. */
4345	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4346	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4347	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4348	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4349	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4350	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4351	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4352	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4353	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4354	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4355	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4356	case VMX_VMCS64_CTRL_EPTP_FULL:
4357	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4358	/* 64-bit Read-only data fields. */
4359	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4360	/* 64-bit Guest-state fields. */
4361	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4362	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4363	case VMX_VMCS64_GUEST_PAT_FULL:
4364	case VMX_VMCS64_GUEST_EFER_FULL:
4365	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4366	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4367	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4368	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4369	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4370	/* 64-bit Host-state fields. */
4371	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4372	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4373	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4374	{
4375	rc = VMXWriteVmcs32(idxField, u64Val);
4376	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4377	break;
4378	}
4379
4380	/*
4381	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4382	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4383	*/
4384	/* Natural-width Guest-state fields. */
4385	case VMX_VMCS_GUEST_CR0:
4386	case VMX_VMCS_GUEST_CR3:
4387	case VMX_VMCS_GUEST_CR4:
4388	case VMX_VMCS_GUEST_ES_BASE:
4389	case VMX_VMCS_GUEST_CS_BASE:
4390	case VMX_VMCS_GUEST_SS_BASE:
4391	case VMX_VMCS_GUEST_DS_BASE:
4392	case VMX_VMCS_GUEST_FS_BASE:
4393	case VMX_VMCS_GUEST_GS_BASE:
4394	case VMX_VMCS_GUEST_LDTR_BASE:
4395	case VMX_VMCS_GUEST_TR_BASE:
4396	case VMX_VMCS_GUEST_GDTR_BASE:
4397	case VMX_VMCS_GUEST_IDTR_BASE:
4398	case VMX_VMCS_GUEST_DR7:
4399	case VMX_VMCS_GUEST_RSP:
4400	case VMX_VMCS_GUEST_RIP:
4401	case VMX_VMCS_GUEST_RFLAGS:
4402	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4403	case VMX_VMCS_GUEST_SYSENTER_ESP:
4404	case VMX_VMCS_GUEST_SYSENTER_EIP:
4405	{
4406	if ((u64Val >> 32ULL) == 0)
4407	{
4408	/* If this field is 64-bit, VT-x will zero out the top bits. */
4409	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4410	}
4411	else
4412	{
4413	/* Assert that only the 32->64 switcher case should ever come here. */
4414	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4415	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4416	}
4417	break;
4418	}
4419
4420	default:
4421	{
4422	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4423	rc = VERR_INVALID_PARAMETER;
4424	break;
4425	}
4426	}
4427	AssertRCReturn(rc, rc);
4428	return rc;
4429	}
4430
4431
4432	/**
4433	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4434	* hosts (except darwin) for 64-bit guests.
4435	*
4436	* @param pVCpu Pointer to the VMCPU.
4437	* @param idxField The VMCS field encoding.
4438	* @param u64Val 16, 32 or 64 bits value.
4439	*/
4440	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4441	{
4442	AssertPtr(pVCpu);
4443	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4444
4445	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4446	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4447
4448	/* Make sure there are no duplicates. */
4449	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4450	{
4451	if (pCache->Write.aField[i] == idxField)
4452	{
4453	pCache->Write.aFieldVal[i] = u64Val;
4454	return VINF_SUCCESS;
4455	}
4456	}
4457
4458	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4459	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4460	pCache->Write.cValidEntries++;
4461	return VINF_SUCCESS;
4462	}
4463
4464
4465	/**
4466	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4467	*
4468	* @param pVCpu Pointer to the VMCPU.
4469	* @param pCache Pointer to the VMCS cache.
4470	*
4471	* @remarks No-long-jump zone!!!
4472	*/
4473	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4474	{
4475	AssertPtr(pCache);
4476	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4477	{
4478	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4479	AssertRC(rc);
4480	}
4481	pCache->Write.cValidEntries = 0;
4482	}
4483
4484
4485	/**
4486	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4487	*
4488	* @param pVCpu Pointer to the VMCPU.
4489	* @param pCache Pointer to the VMCS cache.
4490	*
4491	* @remarks No-long-jump zone!!!
4492	*/
4493	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4494	{
4495	AssertPtr(pCache);
4496	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4497	{
4498	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4499	AssertRC(rc);
4500	}
4501	}
4502	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4503
4504
4505	/**
4506	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4507	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4508	* timer.
4509	*
4510	* @returns VBox status code.
4511	* @param pVM Pointer to the VM.
4512	* @param pVCpu Pointer to the VMCPU.
4513	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4514	* out-of-sync. Make sure to update the required fields
4515	* before using them.
4516	* @remarks No-long-jump zone!!!
4517	*/
4518	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4519	{
4520	int rc = VERR_INTERNAL_ERROR_5;
4521	bool fOffsettedTsc = false;
4522	if (pVM->hm.s.vmx.fUsePreemptTimer)
4523	{
4524	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4525
4526	/* Make sure the returned values have sane upper and lower boundaries. */
4527	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4528	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4529	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4530	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4531
4532	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4533	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4534	}
4535	else
4536	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4537
4538	if (fOffsettedTsc)
4539	{
4540	uint64_t u64CurTSC = ASMReadTSC();
4541	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4542	{
4543	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4544	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4545
4546	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4547	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4548	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4549	}
4550	else
4551	{
4552	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4553	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4554	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4555	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4556	}
4557	}
4558	else
4559	{
4560	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4561	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4562	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4563	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4564	}
4565	}
4566
4567
4568	/**
4569	* Determines if an exception is a contributory exception. Contributory
4570	* exceptions are ones which can cause double-faults.
4571	*
4572	* @returns true if the exception is contributory, false otherwise.
4573	* @param uVector The exception vector.
4574	*/
4575	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4576	{
4577	switch (uVector)
4578	{
4579	case X86_XCPT_GP:
4580	case X86_XCPT_SS:
4581	case X86_XCPT_NP:
4582	case X86_XCPT_TS:
4583	case X86_XCPT_DE:
4584	return true;
4585	default:
4586	break;
4587	}
4588	return false;
4589	}
4590
4591
4592	/**
4593	* Determines if an exception is a benign exception. Benign exceptions
4594	* are ones which cannot cause double-faults.
4595	*
4596	* @returns true if the exception is benign, false otherwise.
4597	* @param uVector The exception vector.
4598	*/
4599	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4600	{
4601	if ( uVector == X86_XCPT_PF
4602	\|\| uVector == X86_XCPT_DF)
4603	{
4604	return false;
4605	}
4606
4607	bool fIsBenignXcpt = !hmR0VmxIsContributoryXcpt(uVector);
4608	#ifdef VBOX_STRICT
4609	switch (uVector)
4610	{
4611	case X86_XCPT_DB:
4612	case X86_XCPT_NMI:
4613	case X86_XCPT_BP:
4614	case X86_XCPT_OF:
4615	case X86_XCPT_BR:
4616	case X86_XCPT_UD:
4617	case X86_XCPT_NM:
4618	case X86_XCPT_CO_SEG_OVERRUN:
4619	case X86_XCPT_MF:
4620	case X86_XCPT_AC:
4621	case X86_XCPT_MC:
4622	case X86_XCPT_XF:
4623	AssertMsg(fIsBenignXcpt, ("%#x\n", uVector));
4624	break;
4625	default:
4626	AssertMsg(!fIsBenignXcpt, ("%#x\n", uVector));
4627	break;
4628	}
4629	#endif
4630	return fIsBenignXcpt;
4631	}
4632
4633
4634	/**
4635	* Sets an event as a pending event to be injected into the guest.
4636	*
4637	* @param pVCpu Pointer to the VMCPU.
4638	* @param u32IntrInfo The VM-entry interruption-information field.
4639	* @param cbInstr The VM-entry instruction length in bytes (for software
4640	* interrupts, exceptions and privileged software
4641	* exceptions).
4642	* @param u32ErrCode The VM-entry exception error code.
4643	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4644	* page-fault.
4645	*/
4646	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4647	RTGCUINTPTR GCPtrFaultAddress)
4648	{
4649	Assert(!pVCpu->hm.s.Event.fPending);
4650	pVCpu->hm.s.Event.fPending = true;
4651	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4652	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4653	pVCpu->hm.s.Event.cbInstr = cbInstr;
4654	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4655	}
4656
4657
4658	/**
4659	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4660	*
4661	* @param pVCpu Pointer to the VMCPU.
4662	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4663	* out-of-sync. Make sure to update the required fields
4664	* before using them.
4665	*/
4666	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4667	{
4668	/* Inject the double-fault. */
4669	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4670	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4671	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4672	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4673	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4674	}
4675
4676
4677	/**
4678	* Handle a condition that occurred while delivering an event through the guest
4679	* IDT.
4680	*
4681	* @returns VBox status code (informational error codes included).
4682	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4683	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4684	* continue execution of the guest which will delivery the #DF.
4685	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4686	*
4687	* @param pVCpu Pointer to the VMCPU.
4688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4689	* out-of-sync. Make sure to update the required fields
4690	* before using them.
4691	* @param pVmxTransient Pointer to the VMX transient structure.
4692	*
4693	* @remarks No-long-jump zone!!!
4694	*/
4695	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4696	{
4697	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4698	AssertRC(rc);
4699	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4700	{
4701	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4702	AssertRCReturn(rc, rc);
4703
4704	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4705	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4706	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4707
4708	typedef enum
4709	{
4710	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4711	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4712	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4713	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4714	} VMXREFLECTXCPT;
4715
4716	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4717	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4718	RTGCUINTPTR GCPtrFaultAddress = 0;
4719	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4720	{
4721	enmReflect = VMXREFLECTXCPT_XCPT;
4722	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4723	&& uExitVector == X86_XCPT_PF)
4724	{
4725	GCPtrFaultAddress = pMixedCtx->cr2;
4726	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4727	}
4728	else if ( uExitVector == X86_XCPT_PF
4729	&& uIdtVector == X86_XCPT_PF)
4730	{
4731	pVmxTransient->fVectoringPF = true;
4732	GCPtrFaultAddress = pMixedCtx->cr2;
4733	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4734	}
4735	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & VMX_CONTRIBUTORY_XCPT_BITMAP)
4736	&& hmR0VmxIsContributoryXcpt(uExitVector)
4737	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4738	\|\| uIdtVector == X86_XCPT_PF))
4739	{
4740	enmReflect = VMXREFLECTXCPT_DF;
4741	}
4742	else if (uIdtVector == X86_XCPT_DF)
4743	enmReflect = VMXREFLECTXCPT_TF;
4744	}
4745	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4746	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4747	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4748	{
4749	/*
4750	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4751	* (whatever they are) as they reoccur when restarting the instruction.
4752	*/
4753	enmReflect = VMXREFLECTXCPT_XCPT;
4754	}
4755
4756	switch (enmReflect)
4757	{
4758	case VMXREFLECTXCPT_XCPT:
4759	{
4760	uint32_t u32ErrCode = 0;
4761	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4762	{
4763	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4764	AssertRCReturn(rc, rc);
4765	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4766	}
4767	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4768	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4769	rc = VINF_SUCCESS;
4770	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4771	break;
4772	}
4773
4774	case VMXREFLECTXCPT_DF:
4775	{
4776	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4777	rc = VINF_VMX_DOUBLE_FAULT;
4778	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4779	uExitVector));
4780	break;
4781	}
4782
4783	case VMXREFLECTXCPT_TF:
4784	{
4785	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4786	rc = VINF_EM_RESET;
4787	break;
4788	}
4789
4790	default:
4791	Assert(rc == VINF_SUCCESS);
4792	break;
4793	}
4794	}
4795	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4796	return rc;
4797	}
4798
4799
4800	/**
4801	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4802	*
4803	* @returns VBox status code.
4804	* @param pVCpu Pointer to the VMCPU.
4805	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4806	* out-of-sync. Make sure to update the required fields
4807	* before using them.
4808	*
4809	* @remarks No-long-jump zone!!!
4810	*/
4811	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4812	{
4813	int rc = VINF_SUCCESS;
4814	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4815	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4816	{
4817	RTGCUINTREG uVal = 0;
4818	RTCCUINTREG uShadow = 0;
4819	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4820	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4821	AssertRCReturn(rc, rc);
4822	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4823	CPUMSetGuestCR0(pVCpu, uVal);
4824	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4825	}
4826	return rc;
4827	}
4828
4829
4830	/**
4831	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4832	*
4833	* @returns VBox status code.
4834	* @param pVCpu Pointer to the VMCPU.
4835	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4836	* out-of-sync. Make sure to update the required fields
4837	* before using them.
4838	*
4839	* @remarks No-long-jump zone!!!
4840	*/
4841	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4842	{
4843	int rc = VINF_SUCCESS;
4844	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4845	{
4846	RTGCUINTREG uVal = 0;
4847	RTCCUINTREG uShadow = 0;
4848	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4849	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4850	AssertRCReturn(rc, rc);
4851	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4852	CPUMSetGuestCR4(pVCpu, uVal);
4853	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4854	}
4855	return rc;
4856	}
4857
4858
4859	/**
4860	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4861	*
4862	* @returns VBox status code.
4863	* @param pVCpu Pointer to the VMCPU.
4864	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4865	* out-of-sync. Make sure to update the required fields
4866	* before using them.
4867	*
4868	* @remarks No-long-jump zone!!!
4869	*/
4870	DECLINLINE(int) hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4871	{
4872	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4873	return VINF_SUCCESS;
4874
4875	RTGCUINTREG uVal = 0;
4876	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4877	AssertRCReturn(rc, rc);
4878	pMixedCtx->rip = uVal;
4879	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4880	return rc;
4881	}
4882
4883
4884	/**
4885	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4886	*
4887	* @returns VBox status code.
4888	* @param pVCpu Pointer to the VMCPU.
4889	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4890	* out-of-sync. Make sure to update the required fields
4891	* before using them.
4892	*
4893	* @remarks No-long-jump zone!!!
4894	*/
4895	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4896	{
4897	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4898	return VINF_SUCCESS;
4899
4900	RTGCUINTREG uVal = 0;
4901	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4902	AssertRCReturn(rc, rc);
4903	pMixedCtx->rsp = uVal;
4904	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4905	return rc;
4906	}
4907
4908
4909	/**
4910	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4911	*
4912	* @returns VBox status code.
4913	* @param pVCpu Pointer to the VMCPU.
4914	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4915	* out-of-sync. Make sure to update the required fields
4916	* before using them.
4917	*
4918	* @remarks No-long-jump zone!!!
4919	*/
4920	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4921	{
4922	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4923	return VINF_SUCCESS;
4924
4925	RTGCUINTREG uVal = 0;
4926	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4927	AssertRCReturn(rc, rc);
4928	pMixedCtx->rflags.u64 = uVal;
4929
4930	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4931	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4932	{
4933	PVM pVM = pVCpu->CTX_SUFF(pVM);
4934	Assert(pVM->hm.s.vmx.pRealModeTSS);
4935	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4936	pMixedCtx->eflags.Bits.u1VM = 0;
4937	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4938	}
4939
4940	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4941	return rc;
4942	}
4943
4944
4945	/**
4946	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4947	* guest-CPU context.
4948	*/
4949	static int hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4950	{
4951	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4952	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4953	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4954	return rc;
4955	}
4956
4957
4958	/**
4959	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4960	* from the guest-state area in the VMCS.
4961	*
4962	* @param pVCpu Pointer to the VMCPU.
4963	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4964	* out-of-sync. Make sure to update the required fields
4965	* before using them.
4966	*
4967	* @remarks No-long-jump zone!!!
4968	*/
4969	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4970	{
4971	uint32_t uIntrState = 0;
4972	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4973	AssertRC(rc);
4974
4975	if (!uIntrState)
4976	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4977	else
4978	{
4979	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4980	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4981	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4982	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4983	AssertRC(rc);
4984	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4985	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4986	}
4987	}
4988
4989
4990	/**
4991	* Saves the guest's activity state.
4992	*
4993	* @returns VBox status code.
4994	* @param pVCpu Pointer to the VMCPU.
4995	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4996	* out-of-sync. Make sure to update the required fields
4997	* before using them.
4998	*
4999	* @remarks No-long-jump zone!!!
5000	*/
5001	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5002	{
5003	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5004	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
5005	return VINF_SUCCESS;
5006	}
5007
5008
5009	/**
5010	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5011	* the current VMCS into the guest-CPU context.
5012	*
5013	* @returns VBox status code.
5014	* @param pVCpu Pointer to the VMCPU.
5015	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5016	* out-of-sync. Make sure to update the required fields
5017	* before using them.
5018	*
5019	* @remarks No-long-jump zone!!!
5020	*/
5021	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5022	{
5023	int rc = VINF_SUCCESS;
5024	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5025	{
5026	uint32_t u32Val = 0;
5027	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5028	pMixedCtx->SysEnter.cs = u32Val;
5029	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5030	}
5031
5032	RTGCUINTREG uGCVal = 0;
5033	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5034	{
5035	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5036	pMixedCtx->SysEnter.eip = uGCVal;
5037	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5038	}
5039	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5040	{
5041	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5042	pMixedCtx->SysEnter.esp = uGCVal;
5043	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5044	}
5045	return rc;
5046	}
5047
5048
5049	/**
5050	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5051	* context.
5052	*
5053	* @returns VBox status code.
5054	* @param pVCpu Pointer to the VMCPU.
5055	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5056	* out-of-sync. Make sure to update the required fields
5057	* before using them.
5058	*
5059	* @remarks No-long-jump zone!!!
5060	*/
5061	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5062	{
5063	RTGCUINTREG uVal = 0;
5064	int rc = VINF_SUCCESS;
5065	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5066	{
5067	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5068	pMixedCtx->fs.u64Base = uVal;
5069	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5070	}
5071	return rc;
5072	}
5073
5074
5075	/**
5076	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5077	* context.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu Pointer to the VMCPU.
5081	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5082	* out-of-sync. Make sure to update the required fields
5083	* before using them.
5084	*
5085	* @remarks No-long-jump zone!!!
5086	*/
5087	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5088	{
5089	RTGCUINTREG uVal = 0;
5090	int rc = VINF_SUCCESS;
5091	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5092	{
5093	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5094	pMixedCtx->gs.u64Base = uVal;
5095	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5096	}
5097	return rc;
5098	}
5099
5100
5101	/**
5102	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5103	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5104	*
5105	* @returns VBox status code.
5106	* @param pVCpu Pointer to the VMCPU.
5107	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5108	* out-of-sync. Make sure to update the required fields
5109	* before using them.
5110	*
5111	* @remarks No-long-jump zone!!!
5112	*/
5113	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114	{
5115	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5116	return VINF_SUCCESS;
5117
5118	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5119	{
5120	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5121	pMsr += i;
5122	switch (pMsr->u32IndexMSR)
5123	{
5124	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5125	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5126	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5127	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5128	#if 0
5129	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5130	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5131	#endif
5132	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5133	default:
5134	{
5135	AssertFailed();
5136	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5137	}
5138	}
5139	}
5140	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5141	return VINF_SUCCESS;
5142	}
5143
5144
5145	/**
5146	* Saves the guest control registers from the current VMCS into the guest-CPU
5147	* context.
5148	*
5149	* @returns VBox status code.
5150	* @param pVCpu Pointer to the VMCPU.
5151	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5152	* out-of-sync. Make sure to update the required fields
5153	* before using them.
5154	*
5155	* @remarks No-long-jump zone!!!
5156	*/
5157	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5158	{
5159	RTGCUINTREG uVal = 0;
5160	RTGCUINTREG uShadow = 0;
5161	int rc = VINF_SUCCESS;
5162
5163	/* Guest CR0. Guest FPU. */
5164	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5165
5166	/* Guest CR4. */
5167	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5168	AssertRCReturn(rc, rc);
5169
5170	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5171	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5172	{
5173	PVM pVM = pVCpu->CTX_SUFF(pVM);
5174	if ( pVM->hm.s.fNestedPaging
5175	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5176	{
5177	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5178	if (pMixedCtx->cr3 != uVal)
5179	{
5180	CPUMSetGuestCR3(pVCpu, uVal);
5181	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5182	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5183	}
5184
5185	/* We require EFER to check PAE mode. */
5186	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5187
5188	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5189	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5190	{
5191	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5192	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5193	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5194	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5195	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5196	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5197	}
5198	AssertRCReturn(rc, rc);
5199	}
5200
5201	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5202	}
5203	return rc;
5204	}
5205
5206
5207	/**
5208	* Reads a guest segment register from the current VMCS into the guest-CPU
5209	* context.
5210	*
5211	* @returns VBox status code.
5212	* @param pVCpu Pointer to the VMCPU.
5213	* @param idxSel Index of the selector in the VMCS.
5214	* @param idxLimit Index of the segment limit in the VMCS.
5215	* @param idxBase Index of the segment base in the VMCS.
5216	* @param idxAccess Index of the access rights of the segment in the VMCS.
5217	* @param pSelReg Pointer to the segment selector.
5218	*
5219	* @remarks No-long-jump zone!!!
5220	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5221	* as that takes care of whether to read from the VMCS cache or not.
5222	*/
5223	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5224	PCPUMSELREG pSelReg)
5225	{
5226	uint32_t u32Val = 0;
5227	int rc = VMXReadVmcs32(idxSel, &u32Val);
5228	pSelReg->Sel = (uint16_t)u32Val;
5229	pSelReg->ValidSel = (uint16_t)u32Val;
5230	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5231
5232	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5233	pSelReg->u32Limit = u32Val;
5234
5235	RTGCUINTREG uGCVal = 0;
5236	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5237	pSelReg->u64Base = uGCVal;
5238
5239	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5240	pSelReg->Attr.u = u32Val;
5241	AssertRCReturn(rc, rc);
5242
5243	/*
5244	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5245	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5246	*/
5247	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5248	{
5249	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5250	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5251	}
5252	return rc;
5253	}
5254
5255
5256	/**
5257	* Saves the guest segment registers from the current VMCS into the guest-CPU
5258	* context.
5259	*
5260	* @returns VBox status code.
5261	* @param pVCpu Pointer to the VMCPU.
5262	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5263	* out-of-sync. Make sure to update the required fields
5264	* before using them.
5265	*
5266	* @remarks No-long-jump zone!!!
5267	*/
5268	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5269	{
5270	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5271	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5272	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5273	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5274	#else
5275	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5276	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5277	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5278	#endif
5279
5280	int rc = VINF_SUCCESS;
5281
5282	/* Guest segment registers. */
5283	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5284	{
5285	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5286	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5287	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5288	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5289	rc \|= VMXLOCAL_READ_SEG(ES, es);
5290	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5291	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5292	AssertRCReturn(rc, rc);
5293
5294	/* Restore segment attributes for real-on-v86 mode hack. */
5295	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5296	{
5297	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5298	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5299	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5300	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5301	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5302	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5303	}
5304	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5305	}
5306
5307	/* Guest LDTR. */
5308	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5309	{
5310	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5311	AssertRCReturn(rc, rc);
5312	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5313	}
5314
5315	/* Guest GDTR. */
5316	RTGCUINTREG uGCVal = 0;
5317	uint32_t u32Val = 0;
5318	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5319	{
5320	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5321	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5322	pMixedCtx->gdtr.pGdt = uGCVal;
5323	pMixedCtx->gdtr.cbGdt = u32Val;
5324	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5325	}
5326
5327	/* Guest IDTR. */
5328	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5329	{
5330	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5331	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5332	pMixedCtx->idtr.pIdt = uGCVal;
5333	pMixedCtx->idtr.cbIdt = u32Val;
5334	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5335	}
5336
5337	/* Guest TR. */
5338	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5339	{
5340	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5341
5342	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5343	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5344	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5345	AssertRCReturn(rc, rc);
5346	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5347	}
5348	return rc;
5349	}
5350
5351
5352	/**
5353	* Saves the guest debug registers from the current VMCS into the guest-CPU
5354	* context.
5355	*
5356	* @returns VBox status code.
5357	* @param pVCpu Pointer to the VMCPU.
5358	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5359	* out-of-sync. Make sure to update the required fields
5360	* before using them.
5361	*
5362	* @remarks No-long-jump zone!!!
5363	*/
5364	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5365	{
5366	int rc = VINF_SUCCESS;
5367	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5368	{
5369	RTGCUINTREG uVal;
5370	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5371	pMixedCtx->dr[7] = uVal;
5372
5373	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5374	}
5375	return rc;
5376	}
5377
5378
5379	/**
5380	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5381	*
5382	* @returns VBox status code.
5383	* @param pVCpu Pointer to the VMCPU.
5384	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5385	* out-of-sync. Make sure to update the required fields
5386	* before using them.
5387	*
5388	* @remarks No-long-jump zone!!!
5389	*/
5390	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5391	{
5392	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5393	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5394	return VINF_SUCCESS;
5395	}
5396
5397
5398	/**
5399	* Saves the entire guest state from the currently active VMCS into the
5400	* guest-CPU context. This essentially VMREADs all guest-data.
5401	*
5402	* @returns VBox status code.
5403	* @param pVCpu Pointer to the VMCPU.
5404	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5405	* out-of-sync. Make sure to update the required fields
5406	* before using them.
5407	*/
5408	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5409	{
5410	Assert(pVCpu);
5411	Assert(pMixedCtx);
5412
5413	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5414	return VINF_SUCCESS;
5415
5416	VMMRZCallRing3Disable(pVCpu);
5417	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5418	LogFunc(("\n"));
5419
5420	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5421	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5422
5423	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5424	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5425
5426	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5427	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5428
5429	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5430	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5431
5432	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5433	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5434
5435	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5436	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5437
5438	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5439	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5440
5441	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5442	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5443
5444	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5445	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5446
5447	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5448	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5449
5450	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5451	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5452
5453	VMMRZCallRing3Enable(pVCpu);
5454	return rc;
5455	}
5456
5457
5458	/**
5459	* Check per-VM and per-VCPU force flag actions that require us to go back to
5460	* ring-3 for one reason or another.
5461	*
5462	* @returns VBox status code (information status code included).
5463	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5464	* ring-3.
5465	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5466	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5467	* interrupts)
5468	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5469	* all EMTs to be in ring-3.
5470	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5471	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5472	* to the EM loop.
5473	*
5474	* @param pVM Pointer to the VM.
5475	* @param pVCpu Pointer to the VMCPU.
5476	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5477	* out-of-sync. Make sure to update the required fields
5478	* before using them.
5479	*/
5480	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5481	{
5482	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5483
5484	int rc = VERR_INTERNAL_ERROR_5;
5485	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5486	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5487	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5488	{
5489	/* We need the control registers now, make sure the guest-CPU context is updated. */
5490	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5491	AssertRCReturn(rc, rc);
5492
5493	/* Pending HM CR3 sync. */
5494	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5495	{
5496	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5497	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5498	}
5499	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5500	{
5501	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5502	AssertRC(rc);
5503	}
5504
5505	/* Pending PGM C3 sync. */
5506	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5507	{
5508	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5509	if (rc != VINF_SUCCESS)
5510	{
5511	AssertRC(rc);
5512	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5513	return rc;
5514	}
5515	}
5516
5517	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5518	/* -XXX- what was that about single stepping? */
5519	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5520	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5521	{
5522	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5523	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5524	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5525	return rc;
5526	}
5527
5528	/* Pending VM request packets, such as hardware interrupts. */
5529	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5530	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5531	{
5532	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5533	return VINF_EM_PENDING_REQUEST;
5534	}
5535
5536	/* Pending PGM pool flushes. */
5537	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5538	{
5539	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5540	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5541	}
5542
5543	/* Pending DMA requests. */
5544	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5545	{
5546	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5547	return VINF_EM_RAW_TO_R3;
5548	}
5549	}
5550
5551	/* Paranoia. */
5552	Assert(rc != VERR_EM_INTERPRETER);
5553	return VINF_SUCCESS;
5554	}
5555
5556
5557	/**
5558	* Converts any TRPM trap into a pending VMX event. This is typically used when
5559	* entering from ring-3 (not longjmp returns).
5560	*
5561	* @param pVCpu Pointer to the VMCPU.
5562	* @param pCtx Pointer to the guest-CPU context.
5563	*/
5564	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5565	{
5566	if (!TRPMHasTrap(pVCpu))
5567	{
5568	Assert(!pVCpu->hm.s.Event.fPending);
5569	return;
5570	}
5571
5572	uint8_t uVector;
5573	TRPMEVENT enmTrpmEvent;
5574	RTGCUINT uErrCode;
5575	RTGCUINTPTR GCPtrFaultAddress;
5576	uint8_t cbInstr;
5577
5578	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5579	AssertRC(rc);
5580
5581	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5582	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5583	uint32_t u32ErrCode = uErrCode;
5584	if (enmTrpmEvent == TRPM_TRAP)
5585	{
5586	switch (uVector)
5587	{
5588	case X86_XCPT_BP:
5589	case X86_XCPT_OF:
5590	{
5591	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5592	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5593	break;
5594	}
5595
5596	case X86_XCPT_PF:
5597	pCtx->cr2 = GCPtrFaultAddress;
5598	/* no break */
5599	case X86_XCPT_DF:
5600	case X86_XCPT_TS:
5601	case X86_XCPT_NP:
5602	case X86_XCPT_SS:
5603	case X86_XCPT_GP:
5604	case X86_XCPT_AC:
5605	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5606	them which VT-x/VMM pushes to the guest stack. */
5607	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5608	/* no break! */
5609	default:
5610	{
5611	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5612	break;
5613	}
5614	}
5615	}
5616	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5617	{
5618	if (uVector != X86_XCPT_NMI)
5619	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5620	else
5621	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5622	}
5623	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5624	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5625	else
5626	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5627
5628	rc = TRPMResetTrap(pVCpu);
5629	AssertRC(rc);
5630	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5631	u32IntrInfo, enmTrpmEvent, cbInstr, u32ErrCode, GCPtrFaultAddress));
5632	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5633	}
5634
5635
5636	/**
5637	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5638	* VT-x to execute any instruction.
5639	*
5640	* @param pvCpu Pointer to the VMCPU.
5641	*/
5642	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5643	{
5644	if (pVCpu->hm.s.Event.fPending)
5645	{
5646	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5647	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5648	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5649	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5650
5651	/* If a trap was already pending, we did something wrong! */
5652	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5653
5654	TRPMEVENT enmTrapType;
5655	switch (uVectorType)
5656	{
5657	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5658	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5659	enmTrapType = TRPM_HARDWARE_INT;
5660	break;
5661	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5662	enmTrapType = TRPM_SOFTWARE_INT;
5663	break;
5664	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5665	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5666	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5667	enmTrapType = TRPM_TRAP;
5668	break;
5669	default:
5670	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5671	enmTrapType = TRPM_32BIT_HACK;
5672	break;
5673	}
5674
5675	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5676	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5677	AssertRC(rc);
5678
5679	if (fErrorCodeValid)
5680	TRPMSetErrorCode(pVCpu, uErrorCode);
5681	/* A page-fault exception during a page-fault would become a double-fault. */
5682	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5683	&& uVector == X86_XCPT_PF)
5684	{
5685	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5686	}
5687	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5688	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5689	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5690	{
5691	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5692	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5693	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5694	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5695	}
5696	pVCpu->hm.s.Event.fPending = false;
5697	}
5698	}
5699
5700
5701	/**
5702	* Does the necessary state syncing before doing a longjmp to ring-3.
5703	*
5704	* @param pVM Pointer to the VM.
5705	* @param pVCpu Pointer to the VMCPU.
5706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5707	* out-of-sync. Make sure to update the required fields
5708	* before using them.
5709	* @param rcExit The reason for exiting to ring-3. Can be
5710	* VINF_VMM_UNKNOWN_RING3_CALL.
5711	*
5712	* @remarks No-long-jmp zone!!!
5713	*/
5714	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5715	{
5716	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5717	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5718
5719	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5720	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5721	AssertRC(rc);
5722
5723	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5724	if (CPUMIsGuestFPUStateActive(pVCpu))
5725	{
5726	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5727	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5728	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5729	}
5730
5731	/* Restore debug registers if necessary and resync on next R0 reentry. */
5732	if (CPUMIsGuestDebugStateActive(pVCpu))
5733	{
5734	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5735	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5736	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5737	}
5738	else if (CPUMIsHyperDebugStateActive(pVCpu))
5739	{
5740	CPUMR0LoadHostDebugState(pVM, pVCpu);
5741	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5742	}
5743
5744	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5745	}
5746
5747
5748	/**
5749	* An action requires us to go back to ring-3. This function does the necessary
5750	* steps before we can safely return to ring-3. This is not the same as longjmps
5751	* to ring-3, this is voluntary.
5752	*
5753	* @param pVM Pointer to the VM.
5754	* @param pVCpu Pointer to the VMCPU.
5755	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5756	* out-of-sync. Make sure to update the required fields
5757	* before using them.
5758	* @param rcExit The reason for exiting to ring-3. Can be
5759	* VINF_VMM_UNKNOWN_RING3_CALL.
5760	*/
5761	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5762	{
5763	Assert(pVM);
5764	Assert(pVCpu);
5765	Assert(pMixedCtx);
5766	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5767
5768	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5769	{
5770	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5771	return;
5772	}
5773	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5774	{
5775	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5776	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5777	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5778	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5779	return;
5780	}
5781
5782	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5783	VMMRZCallRing3Disable(pVCpu);
5784	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5785
5786	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5787	hmR0VmxUpdateTRPM(pVCpu);
5788
5789	/* Sync. the guest state. */
5790	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5791	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5792
5793	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5794	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5795	\| CPUM_CHANGED_LDTR
5796	\| CPUM_CHANGED_GDTR
5797	\| CPUM_CHANGED_IDTR
5798	\| CPUM_CHANGED_TR
5799	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5800
5801	/* On our way back from ring-3 the following needs to be done. */
5802	/** @todo This can change with preemption hooks. */
5803	if (rcExit == VINF_EM_RAW_INTERRUPT)
5804	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5805	else
5806	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5807
5808	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5809	VMMRZCallRing3Enable(pVCpu);
5810	}
5811
5812
5813	/**
5814	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5815	* longjump to ring-3 and possibly get preempted.
5816	*
5817	* @param pVCpu Pointer to the VMCPU.
5818	* @param enmOperation The operation causing the ring-3 longjump.
5819	* @param pvUser The user argument (pointer to the possibly
5820	* out-of-date guest-CPU context).
5821	*
5822	* @remarks Must never be called with @a enmOperation ==
5823	* VMMCALLRING3_VM_R0_ASSERTION.
5824	*/
5825	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5826	{
5827	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5828	Assert(pVCpu);
5829	Assert(pvUser);
5830	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5831	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5832
5833	VMMRZCallRing3Disable(pVCpu);
5834	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5835	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5836	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5837	VMMRZCallRing3Enable(pVCpu);
5838	}
5839
5840
5841	/**
5842	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5843	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5844	*
5845	* @returns VBox status code.
5846	* @param pVCpu Pointer to the VMCPU.
5847	*/
5848	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5849	{
5850	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5851	{
5852	/* Enable interrupt-window exiting if it's not in effect already. */
5853	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5854	{
5855	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5856	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5857	AssertRC(rc);
5858	}
5859	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5860	}
5861
5862
5863	/**
5864	* Injects any pending events into the guest if the guest is in a state to
5865	* receive them.
5866	*
5867	* @returns VBox status code (informational status codes included).
5868	* @param pVCpu Pointer to the VMCPU.
5869	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5870	* out-of-sync. Make sure to update the required fields
5871	* before using them.
5872	*/
5873	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5874	{
5875	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5876	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5877	const bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5878	const bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5879
5880	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS));
5881	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5882	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5883	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5884
5885	int rc = VINF_SUCCESS;
5886	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5887	{
5888	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5889	bool fInject = true;
5890	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5891	{
5892	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5893	AssertRCReturn(rc, rc);
5894	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5895	if ( fBlockInt
5896	\|\| fBlockSti
5897	\|\| fBlockMovSS)
5898	{
5899	fInject = false;
5900	}
5901	}
5902	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5903	&& ( fBlockMovSS
5904	\|\| fBlockSti))
5905	{
5906	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5907	fInject = false;
5908	}
5909
5910	if (fInject)
5911	{
5912	Log(("Injecting pending event\n"));
5913	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5914	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5915	AssertRCReturn(rc, rc);
5916	pVCpu->hm.s.Event.fPending = false;
5917	}
5918	else
5919	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5920	} /** @todo SMI. SMIs take priority over NMIs. */
5921	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5922	{
5923	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5924	if ( !fBlockMovSS
5925	&& !fBlockSti)
5926	{
5927	Log(("Injecting NMI\n"));
5928	RTGCUINTPTR uIntrInfo;
5929	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5930	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5931	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5932	AssertRCReturn(rc, rc);
5933	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5934	}
5935	else
5936	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5937	}
5938	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5939	{
5940	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5941	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5942	AssertRCReturn(rc, rc);
5943	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5944	if ( !fBlockInt
5945	&& !fBlockSti
5946	&& !fBlockMovSS)
5947	{
5948	uint8_t u8Interrupt;
5949	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5950	if (RT_SUCCESS(rc))
5951	{
5952	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5953	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5954	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5955	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5956	}
5957	else
5958	{
5959	/** @todo Does this actually happen? If not turn it into an assertion. */
5960	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5961	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5962	rc = VINF_SUCCESS;
5963	}
5964	}
5965	else
5966	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5967	}
5968
5969	/*
5970	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5971	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5972	*/
5973	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5974	AssertRC(rc2);
5975	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5976	return rc;
5977	}
5978
5979
5980	/**
5981	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5982	*
5983	* @param pVCpu Pointer to the VMCPU.
5984	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5985	* out-of-sync. Make sure to update the required fields
5986	* before using them.
5987	*/
5988	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5989	{
5990	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5991	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5992	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5993	}
5994
5995
5996	/**
5997	* Injects a double-fault (#DF) exception into the VM.
5998	*
5999	* @returns VBox status code (informational status code included).
6000	* @param pVCpu Pointer to the VMCPU.
6001	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6002	* out-of-sync. Make sure to update the required fields
6003	* before using them.
6004	*/
6005	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6006	{
6007	/* Inject the double-fault. */
6008	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6009	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6010	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6011	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6012	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6013	}
6014
6015
6016	/**
6017	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6018	*
6019	* @param pVCpu Pointer to the VMCPU.
6020	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6021	* out-of-sync. Make sure to update the required fields
6022	* before using them.
6023	*/
6024	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6025	{
6026	/* Inject the debug-exception. */
6027	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6028	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6029	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6030	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6031	}
6032
6033
6034	/**
6035	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6036	*
6037	* @param pVCpu Pointer to the VMCPU.
6038	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6039	* out-of-sync. Make sure to update the required fields
6040	* before using them.
6041	* @param cbInstr The value of RIP that is to be pushed on the guest
6042	* stack.
6043	*/
6044	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6045	{
6046	/* Inject the overflow exception. */
6047	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6048	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6049	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6050	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6051	}
6052
6053
6054	/**
6055	* Injects a general-protection (#GP) fault into the VM.
6056	*
6057	* @returns VBox status code (informational status code included).
6058	* @param pVCpu Pointer to the VMCPU.
6059	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6060	* out-of-sync. Make sure to update the required fields
6061	* before using them.
6062	* @param u32ErrorCode The error code associated with the #GP.
6063	*/
6064	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6065	uint32_t *puIntrState)
6066	{
6067	/* Inject the general-protection fault. */
6068	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6069	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6070	if (fErrorCodeValid)
6071	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6072	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6073	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6074	}
6075
6076
6077	/**
6078	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6079	*
6080	* @param pVCpu Pointer to the VMCPU.
6081	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6082	* out-of-sync. Make sure to update the required fields
6083	* before using them.
6084	* @param uVector The software interrupt vector number.
6085	* @param cbInstr The value of RIP that is to be pushed on the guest
6086	* stack.
6087	*/
6088	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6089	{
6090	/* Inject the INTn. */
6091	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6092	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6093	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6094	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6095	}
6096
6097
6098	/**
6099	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6100	* stack.
6101	*
6102	* @returns VBox status code (information status code included).
6103	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6104	* @param pVM Pointer to the VM.
6105	* @param pMixedCtx Pointer to the guest-CPU context.
6106	* @param uValue The value to push to the guest stack.
6107	*/
6108	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6109	{
6110	/*
6111	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6112	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6113	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6114	*/
6115	if (pMixedCtx->sp == 1)
6116	return VINF_EM_RESET;
6117	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6118	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6119	AssertRCReturn(rc, rc);
6120	return rc;
6121	}
6122
6123
6124	/**
6125	* Injects an event into the guest upon VM-entry by updating the relevant fields
6126	* in the VM-entry area in the VMCS.
6127	*
6128	* @returns VBox status code (informational error codes included).
6129	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6130	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6131	*
6132	* @param pVCpu Pointer to the VMCPU.
6133	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6134	* out-of-sync. Make sure to update the required fields
6135	* before using them.
6136	* @param u64IntrInfo The VM-entry interruption-information field.
6137	* @param cbInstr The VM-entry instruction length in bytes (for software
6138	* interrupts, exceptions and privileged software
6139	* exceptions).
6140	* @param u32ErrCode The VM-entry exception error code.
6141	* @param puIntrState Pointer to the current guest interruptibility-state.
6142	* This interruptibility-state will be updated if
6143	* necessary. This cannot not be NULL.
6144	*
6145	* @remarks No-long-jump zone!!!
6146	* @remarks Requires CR0!
6147	*/
6148	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6149	uint32_t u32ErrCode, uint32_t *puIntrState)
6150	{
6151	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6152	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6153	Assert(puIntrState);
6154	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6155
6156	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6157	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6158
6159	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6160	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6161	\|\| !((*puIntrState) & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6162
6163	/* We require CR0 to check if the guest is in real-mode. */
6164	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6165	AssertRCReturn(rc, rc);
6166
6167	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6168
6169	/*
6170	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6171	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6172	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6173	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6174	*/
6175	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6176	{
6177	PVM pVM = pVCpu->CTX_SUFF(pVM);
6178	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6179	{
6180	Assert(PDMVmmDevHeapIsEnabled(pVM));
6181	Assert(pVM->hm.s.vmx.pRealModeTSS);
6182
6183	/* Save the required guest state bits from the VMCS. */
6184	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6185	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6186	AssertRCReturn(rc, rc);
6187
6188	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6189	const size_t cbIdtEntry = 4;
6190	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6191	{
6192	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6193	if (uVector == X86_XCPT_DF)
6194	return VINF_EM_RESET;
6195	else if (uVector == X86_XCPT_GP)
6196	{
6197	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6198	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6199	}
6200
6201	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6202	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6203	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6204	}
6205
6206	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6207	uint16_t uGuestIp = pMixedCtx->ip;
6208	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6209	{
6210	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6211	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6212	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6213	}
6214	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6215	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6216
6217	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6218	uint16_t offIdtEntry = 0;
6219	RTSEL selIdtEntry = 0;
6220	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6221	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6222	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6223	AssertRCReturn(rc, rc);
6224
6225	/* Construct the stack frame for the interrupt/exception handler. */
6226	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6227	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6228	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6229	AssertRCReturn(rc, rc);
6230
6231	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6232	if (rc == VINF_SUCCESS)
6233	{
6234	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6235	pMixedCtx->rip = offIdtEntry;
6236	pMixedCtx->cs.Sel = selIdtEntry;
6237	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6238	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6239	\| HM_CHANGED_GUEST_RIP
6240	\| HM_CHANGED_GUEST_RFLAGS
6241	\| HM_CHANGED_GUEST_RSP;
6242
6243	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6244	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6245	{
6246	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6247	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6248	Log(("Clearing inhibition due to STI.\n"));
6249	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6250	}
6251	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6252	}
6253	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6254	return rc;
6255	}
6256	else
6257	{
6258	/*
6259	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6260	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6261	*/
6262	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6263	}
6264	}
6265
6266	/* Validate. */
6267	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6268	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6269	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6270
6271	/* Inject. */
6272	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6273	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6274	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6275	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6276	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6277	AssertRCReturn(rc, rc);
6278	return rc;
6279	}
6280
6281
6282	/**
6283	* Enters the VT-x session.
6284	*
6285	* @returns VBox status code.
6286	* @param pVM Pointer to the VM.
6287	* @param pVCpu Pointer to the VMCPU.
6288	* @param pCpu Pointer to the CPU info struct.
6289	*/
6290	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6291	{
6292	AssertPtr(pVM);
6293	AssertPtr(pVCpu);
6294	Assert(pVM->hm.s.vmx.fSupported);
6295	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6296	NOREF(pCpu);
6297
6298	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6299
6300	/* Make sure we're in VMX root mode. */
6301	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6302	if (!(u32HostCR4 & X86_CR4_VMXE))
6303	{
6304	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6305	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6306	}
6307
6308	/* Load the active VMCS as the current one. */
6309	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6310	if (RT_FAILURE(rc))
6311	return rc;
6312
6313	/** @todo this will change with preemption hooks where can can VMRESUME as long
6314	* as we're no preempted. */
6315	pVCpu->hm.s.fResumeVM = false;
6316	return VINF_SUCCESS;
6317	}
6318
6319
6320	/**
6321	* Leaves the VT-x session.
6322	*
6323	* @returns VBox status code.
6324	* @param pVM Pointer to the VM.
6325	* @param pVCpu Pointer to the VMCPU.
6326	* @param pCtx Pointer to the guest-CPU context.
6327	*/
6328	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6329	{
6330	AssertPtr(pVCpu);
6331	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6332	NOREF(pVM);
6333	NOREF(pCtx);
6334
6335	/** @todo this will change with preemption hooks where we only VMCLEAR when
6336	* we are actually going to be preempted, not all the time like we
6337	* currently do. */
6338	/*
6339	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6340	* and mark the VMCS launch-state as "clear".
6341	*/
6342	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6343	return rc;
6344	}
6345
6346
6347	/**
6348	* Saves the host state in the VMCS host-state.
6349	* Sets up the VM-exit MSR-load area.
6350	*
6351	* The CPU state will be loaded from these fields on every successful VM-exit.
6352	*
6353	* @returns VBox status code.
6354	* @param pVM Pointer to the VM.
6355	* @param pVCpu Pointer to the VMCPU.
6356	*
6357	* @remarks No-long-jump zone!!!
6358	*/
6359	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6360	{
6361	AssertPtr(pVM);
6362	AssertPtr(pVCpu);
6363	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6364
6365	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6366
6367	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6368	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6369	return VINF_SUCCESS;
6370
6371	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6372	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6373
6374	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6375	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6376
6377	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6378	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6379
6380	/* Reset the host-state-changed flag. */
6381	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6382	return rc;
6383	}
6384
6385
6386	/**
6387	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6388	* loaded from these fields on every successful VM-entry.
6389	*
6390	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6391	* Sets up the VM-entry controls.
6392	* Sets up the appropriate VMX non-root function to execute guest code based on
6393	* the guest CPU mode.
6394	*
6395	* @returns VBox status code.
6396	* @param pVM Pointer to the VM.
6397	* @param pVCpu Pointer to the VMCPU.
6398	* @param pCtx Pointer to the guest-CPU context.
6399	*
6400	* @remarks No-long-jump zone!!!
6401	*/
6402	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6403	{
6404	AssertPtr(pVM);
6405	AssertPtr(pVCpu);
6406	AssertPtr(pCtx);
6407	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6408
6409	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6410
6411	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6412
6413	/* Determine real-on-v86 mode. */
6414	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6415	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6416	&& CPUMIsGuestInRealModeEx(pCtx))
6417	{
6418	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6419	}
6420
6421	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6422	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6423
6424	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6425	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6426
6427	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6428	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6429
6430	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6431	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6432
6433	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6434	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6435
6436	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6437	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6438
6439	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6440	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6441
6442	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6443	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6444
6445	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6446	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6447
6448	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6449	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6450
6451	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6452	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6453	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6454
6455	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6456	return rc;
6457	}
6458
6459
6460	/**
6461	* Does the preparations before executing guest code in VT-x.
6462	*
6463	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6464	* recompiler. We must be cautious what we do here regarding committing
6465	* guest-state information into the the VMCS assuming we assuredly execute the
6466	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6467	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6468	* that the recompiler can (and should) use them when it resumes guest
6469	* execution. Otherwise such operations must be done when we can no longer
6470	* exit to ring-3.
6471	*
6472	* @returns VBox status code (informational status codes included).
6473	* @retval VINF_SUCCESS if we can proceed with running the guest.
6474	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6475	* into the guest.
6476	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6477	*
6478	* @param pVM Pointer to the VM.
6479	* @param pVCpu Pointer to the VMCPU.
6480	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6481	* out-of-sync. Make sure to update the required fields
6482	* before using them.
6483	* @param pVmxTransient Pointer to the VMX transient structure.
6484	*
6485	* @remarks Called with preemption disabled.
6486	*/
6487	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6488	{
6489	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6490
6491	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6492	PGMRZDynMapFlushAutoSet(pVCpu);
6493	#endif
6494
6495	/* Check force flag actions that might require us to go back to ring-3. */
6496	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6497	if (rc != VINF_SUCCESS)
6498	return rc;
6499
6500	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6501	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6502	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6503	{
6504	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6505	RTGCPHYS GCPhysApicBase;
6506	GCPhysApicBase = pMixedCtx->msrApicBase;
6507	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6508
6509	/* Unalias any existing mapping. */
6510	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6511	AssertRCReturn(rc, rc);
6512
6513	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6514	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6515	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6516	AssertRCReturn(rc, rc);
6517
6518	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6519	}
6520
6521	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6522	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6523	pVmxTransient->uEFlags = ASMIntDisableFlags();
6524	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6525	{
6526	ASMSetFlags(pVmxTransient->uEFlags);
6527	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6528	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6529	return VINF_EM_RAW_INTERRUPT;
6530	}
6531	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6532	#endif
6533
6534	/*
6535	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6536	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6537	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6538	*/
6539	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6540	AssertRCReturn(rc, rc);
6541	return rc;
6542	}
6543
6544
6545	/**
6546	* Prepares to run guest code in VT-x and we've committed to doing so. This
6547	* means there is no backing out to ring-3 or anywhere else at this
6548	* point.
6549	*
6550	* @param pVM Pointer to the VM.
6551	* @param pVCpu Pointer to the VMCPU.
6552	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6553	* out-of-sync. Make sure to update the required fields
6554	* before using them.
6555	* @param pVmxTransient Pointer to the VMX transient structure.
6556	*
6557	* @remarks Called with preemption disabled.
6558	* @remarks No-long-jump zone!!!
6559	*/
6560	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6561	{
6562	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6563	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6564
6565	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6566	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6567	pVmxTransient->uEFlags = ASMIntDisableFlags();
6568	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6569	#endif
6570
6571	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6572	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6573	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6574	#ifdef VBOX_ALWAYS_SYNC_FULL_VTX_STATE
6575	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6576	#endif
6577	int rc = VINF_SUCCESS;
6578	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6579	{
6580	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6581	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6582	}
6583	else if (pVCpu->hm.s.fContextUseFlags)
6584	{
6585	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6586	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6587	}
6588	AssertRC(rc);
6589	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6590
6591	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6592	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6593	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6594
6595	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6596	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6597
6598	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6599	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6600	{
6601	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6602	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6603	}
6604
6605	/*
6606	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6607	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6608	*/
6609	if (pVM->hm.s.fTPRPatchingActive)
6610	{
6611	Assert(!CPUMIsGuestInLongMode(pVCpu));
6612
6613	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6614	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6615	AssertRC(rc);
6616
6617	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6618	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6619	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6620	see hmR0VmxLoadGuestApicState(). */
6621	}
6622
6623	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6624	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6625	to start executing. */
6626	}
6627
6628
6629	/**
6630	* Performs some essential restoration of state after running guest code in
6631	* VT-x.
6632	*
6633	* @param pVM Pointer to the VM.
6634	* @param pVCpu Pointer to the VMCPU.
6635	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6636	* out-of-sync. Make sure to update the required fields
6637	* before using them.
6638	* @param pVmxTransient Pointer to the VMX transient structure.
6639	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6640	*
6641	* @remarks Called with interrupts disabled.
6642	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6643	* unconditionally when it is safe to do so.
6644	*/
6645	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6646	{
6647	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6648	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6649
6650	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6651	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6652	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6653	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6654	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6655
6656	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6657	{
6658	/** @todo Find a way to fix hardcoding a guestimate. */
6659	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6660	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6661	}
6662
6663	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6664	Assert(!(ASMGetFlags() & X86_EFL_IF));
6665	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6666
6667	/* Restore the effects of TPR patching if any. */
6668	if (pVM->hm.s.fTPRPatchingActive)
6669	{
6670	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6671	AssertRC(rc);
6672	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6673	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6674	}
6675
6676	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6677	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6678
6679	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6680	uint32_t uExitReason;
6681	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6682	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6683	AssertRC(rc);
6684	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6685	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6686
6687	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6688	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6689
6690	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6691	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6692	{
6693	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6694	return;
6695	}
6696
6697	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6698	{
6699	/* Update the guest interruptibility-state from the VMCS. */
6700	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6701	#if defined(VBOX_ALWAYS_SYNC_FULL_VTX_STATE) \|\| defined(VBOX_ALWAYS_SAVE_FULL_VTX_STATE)
6702	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6703	AssertRC(rc);
6704	#endif
6705	/*
6706	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6707	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6708	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6709	*/
6710	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6711	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6712	{
6713	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6714	AssertRC(rc);
6715	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6716	}
6717	}
6718	}
6719
6720
6721	/**
6722	* Runs the guest code using VT-x.
6723	*
6724	* @returns VBox status code.
6725	* @param pVM Pointer to the VM.
6726	* @param pVCpu Pointer to the VMCPU.
6727	* @param pCtx Pointer to the guest-CPU context.
6728	*
6729	* @remarks Called with preemption disabled.
6730	*/
6731	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6732	{
6733	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6734	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6735
6736	VMXTRANSIENT VmxTransient;
6737	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6738	int rc = VERR_INTERNAL_ERROR_5;
6739	uint32_t cLoops = 0;
6740	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6741
6742	for (;; cLoops++)
6743	{
6744	Assert(!HMR0SuspendPending());
6745	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6746	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6747	(unsigned)RTMpCpuId(), cLoops));
6748
6749	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6750	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6751	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6752	if (rc != VINF_SUCCESS)
6753	break;
6754
6755	/*
6756	* No longjmps to ring-3 from this point on!!!
6757	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6758	* This also disables flushing of the R0-logger instance (if any).
6759	*/
6760	VMMRZCallRing3Disable(pVCpu);
6761	VMMRZCallRing3RemoveNotification(pVCpu);
6762	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6763
6764	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6765	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6766
6767	/*
6768	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6769	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6770	*/
6771	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6772	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6773	{
6774	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6775	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6776	return rc;
6777	}
6778
6779	/* Handle the VM-exit. */
6780	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6781	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6782	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6783	rc = (*g_apfnVMExitHandlers[VmxTransient.uExitReason])(pVCpu, pCtx, &VmxTransient);
6784	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6785	if (rc != VINF_SUCCESS)
6786	break;
6787	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6788	{
6789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6790	rc = VINF_EM_RAW_INTERRUPT;
6791	break;
6792	}
6793	}
6794
6795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6796	if (rc == VERR_EM_INTERPRETER)
6797	rc = VINF_EM_RAW_EMULATE_INSTR;
6798	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6799	return rc;
6800	}
6801
6802	#if 1
6803	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6804	{
6805	int rc;
6806	switch (rcReason)
6807	{
6808	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6809	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6811	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6812	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6813	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6814	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6815	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6820	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6841
6842	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6851
6852	case VMX_EXIT_VMCALL:
6853	case VMX_EXIT_VMCLEAR:
6854	case VMX_EXIT_VMLAUNCH:
6855	case VMX_EXIT_VMPTRLD:
6856	case VMX_EXIT_VMPTRST:
6857	case VMX_EXIT_VMREAD:
6858	case VMX_EXIT_VMRESUME:
6859	case VMX_EXIT_VMWRITE:
6860	case VMX_EXIT_VMXOFF:
6861	case VMX_EXIT_VMXON:
6862	case VMX_EXIT_INVEPT:
6863	case VMX_EXIT_INVVPID:
6864	case VMX_EXIT_VMFUNC:
6865	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6866	break;
6867	default:
6868	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6869	break;
6870	}
6871	return rc;
6872	}
6873	#endif
6874
6875	#ifdef DEBUG
6876	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6877	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6878	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6879	# define VMX_ASSERT_PREEMPT_CPUID() \
6880	do \
6881	{ \
6882	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6883	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6884	} while (0)
6885
6886	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6887	do { \
6888	AssertPtr(pVCpu); \
6889	AssertPtr(pMixedCtx); \
6890	AssertPtr(pVmxTransient); \
6891	Assert(pVmxTransient->fVMEntryFailed == false); \
6892	Assert(ASMIntAreEnabled()); \
6893	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6894	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6895	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6896	(unsigned)pVCpu->idCpu)); \
6897	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6898	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6899	VMX_ASSERT_PREEMPT_CPUID(); \
6900	} while (0)
6901	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6902	do { \
6903	LogFunc(("\n")); \
6904	} while(0)
6905	#else /* Release builds */
6906	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6907	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6908	#endif
6909
6910
6911	/**
6912	* Advances the guest RIP after reading it from the VMCS.
6913	*
6914	* @returns VBox status code.
6915	* @param pVCpu Pointer to the VMCPU.
6916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6917	* out-of-sync. Make sure to update the required fields
6918	* before using them.
6919	* @param pVmxTransient Pointer to the VMX transient structure.
6920	*
6921	* @remarks No-long-jump zone!!!
6922	*/
6923	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6924	{
6925	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6926	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6927	AssertRCReturn(rc, rc);
6928
6929	pMixedCtx->rip += pVmxTransient->cbInstr;
6930	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6931	return rc;
6932	}
6933
6934
6935	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6936	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6937	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6938	/**
6939	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6940	*/
6941	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6942	{
6943	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6944	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6945	return VINF_SUCCESS;
6946	}
6947
6948
6949	/**
6950	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6951	*/
6952	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6953	{
6954	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6955	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6956
6957	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6958	AssertRCReturn(rc, rc);
6959
6960	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6961	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6962	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6963
6964	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6965	{
6966	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6967	return VINF_EM_RAW_INTERRUPT;
6968	}
6969
6970	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6971	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6972	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6973	{
6974	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6975	return VINF_SUCCESS;
6976	}
6977	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6978	{
6979	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6980	return rc;
6981	}
6982
6983	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6984	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6985	switch (uIntrType)
6986	{
6987	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6988	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6989	/* no break */
6990	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6991	{
6992	switch (uVector)
6993	{
6994	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6995	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6996	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6997	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6998	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6999	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7000	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7001	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7002	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7003	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7004	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7005	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7006	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7007	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7008	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7009	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7010	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7011	#endif
7012	default:
7013	{
7014	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7015	AssertRCReturn(rc, rc);
7016
7017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7018	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7019	{
7020	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7021	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7022	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7023	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7024	AssertRCReturn(rc, rc);
7025	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7026	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7027	0 /* GCPtrFaultAddress */);
7028	AssertRCReturn(rc, rc);
7029	}
7030	else
7031	{
7032	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7033	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7034	}
7035	break;
7036	}
7037	}
7038	break;
7039	}
7040
7041	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7042	default:
7043	{
7044	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7045	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7046	break;
7047	}
7048	}
7049	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7050	return rc;
7051	}
7052
7053
7054	/**
7055	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7056	*/
7057	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7058	{
7059	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7060
7061	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7062	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7063	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7064	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7065	AssertRCReturn(rc, rc);
7066
7067	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7068	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7069	return VINF_SUCCESS;
7070	}
7071
7072
7073	/**
7074	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7075	*/
7076	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7077	{
7078	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7079	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7080	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7081	}
7082
7083
7084	/**
7085	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7086	*/
7087	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7088	{
7089	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7091	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7092	}
7093
7094
7095	/**
7096	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7097	*/
7098	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7099	{
7100	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7102	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7103	}
7104
7105
7106	/**
7107	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7108	*/
7109	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7110	{
7111	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7112	PVM pVM = pVCpu->CTX_SUFF(pVM);
7113	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7114	if (RT_LIKELY(rc == VINF_SUCCESS))
7115	{
7116	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7117	Assert(pVmxTransient->cbInstr == 2);
7118	}
7119	else
7120	{
7121	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7122	rc = VERR_EM_INTERPRETER;
7123	}
7124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7125	return rc;
7126	}
7127
7128
7129	/**
7130	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7131	*/
7132	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7133	{
7134	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7135	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7136	AssertRCReturn(rc, rc);
7137
7138	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7139	return VINF_EM_RAW_EMULATE_INSTR;
7140
7141	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7142	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7143	}
7144
7145
7146	/**
7147	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7148	*/
7149	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7150	{
7151	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7152	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7153	AssertRCReturn(rc, rc);
7154
7155	PVM pVM = pVCpu->CTX_SUFF(pVM);
7156	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7157	if (RT_LIKELY(rc == VINF_SUCCESS))
7158	{
7159	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7160	Assert(pVmxTransient->cbInstr == 2);
7161	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7162	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7163	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7164	}
7165	else
7166	{
7167	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7168	rc = VERR_EM_INTERPRETER;
7169	}
7170	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7171	return rc;
7172	}
7173
7174
7175	/**
7176	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7177	*/
7178	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7179	{
7180	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7181	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7182	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7183	AssertRCReturn(rc, rc);
7184
7185	PVM pVM = pVCpu->CTX_SUFF(pVM);
7186	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7187	if (RT_LIKELY(rc == VINF_SUCCESS))
7188	{
7189	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7190	Assert(pVmxTransient->cbInstr == 3);
7191	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7192	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7193	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7194	}
7195	else
7196	{
7197	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7198	rc = VERR_EM_INTERPRETER;
7199	}
7200	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7201	return rc;
7202	}
7203
7204
7205	/**
7206	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7207	*/
7208	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7209	{
7210	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7211	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7212	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7213	AssertRCReturn(rc, rc);
7214
7215	PVM pVM = pVCpu->CTX_SUFF(pVM);
7216	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7217	if (RT_LIKELY(rc == VINF_SUCCESS))
7218	{
7219	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7220	Assert(pVmxTransient->cbInstr == 2);
7221	}
7222	else
7223	{
7224	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7225	rc = VERR_EM_INTERPRETER;
7226	}
7227	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7228	return rc;
7229	}
7230
7231
7232	/**
7233	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7234	*/
7235	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7236	{
7237	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7238	PVM pVM = pVCpu->CTX_SUFF(pVM);
7239	Assert(!pVM->hm.s.fNestedPaging);
7240
7241	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7242	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7243	AssertRCReturn(rc, rc);
7244
7245	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7246	rc = VBOXSTRICTRC_VAL(rc2);
7247	if (RT_LIKELY(rc == VINF_SUCCESS))
7248	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7249	else
7250	{
7251	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7252	pVmxTransient->uExitQualification, rc));
7253	}
7254	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7255	return rc;
7256	}
7257
7258
7259	/**
7260	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7261	*/
7262	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7263	{
7264	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7265	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7266	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7267	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7268	AssertRCReturn(rc, rc);
7269
7270	PVM pVM = pVCpu->CTX_SUFF(pVM);
7271	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7272	if (RT_LIKELY(rc == VINF_SUCCESS))
7273	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7274	else
7275	{
7276	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7277	rc = VERR_EM_INTERPRETER;
7278	}
7279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7280	return rc;
7281	}
7282
7283
7284	/**
7285	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7286	*/
7287	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7288	{
7289	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7290	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7291	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7292	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7293	AssertRCReturn(rc, rc);
7294
7295	PVM pVM = pVCpu->CTX_SUFF(pVM);
7296	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7297	rc = VBOXSTRICTRC_VAL(rc2);
7298	if (RT_LIKELY( rc == VINF_SUCCESS
7299	\|\| rc == VINF_EM_HALT))
7300	{
7301	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7302	AssertRCReturn(rc3, rc3);
7303
7304	if ( rc == VINF_EM_HALT
7305	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7306	{
7307	rc = VINF_SUCCESS;
7308	}
7309	}
7310	else
7311	{
7312	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7313	rc = VERR_EM_INTERPRETER;
7314	}
7315	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7316	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7318	return rc;
7319	}
7320
7321
7322	/**
7323	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7324	*/
7325	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7326	{
7327	/*
7328	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7329	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7330	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7331	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7332	*/
7333	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7334	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7335	}
7336
7337
7338	/**
7339	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7340	*/
7341	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342	{
7343	/*
7344	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7345	* root operation. If we get there there is something funny going on.
7346	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7347	*/
7348	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7349	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7350	}
7351
7352
7353	/**
7354	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7355	*/
7356	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7357	{
7358	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7359	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7360	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7361	}
7362
7363
7364	/**
7365	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7366	*/
7367	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7368	{
7369	/*
7370	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7371	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7372	* See Intel spec. 25.3 "Other Causes of VM-exits".
7373	*/
7374	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7375	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7376	}
7377
7378
7379	/**
7380	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7381	* VM-exit.
7382	*/
7383	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7384	{
7385	/*
7386	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7387	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7388	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7389	*/
7390	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7391	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7397	* VM-exit.
7398	*/
7399	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7400	{
7401	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7402	return VINF_EM_RESET;
7403	}
7404
7405
7406	/**
7407	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7408	*/
7409	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7410	{
7411	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7412	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7413	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7414	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7415	AssertRCReturn(rc, rc);
7416
7417	pMixedCtx->rip++;
7418	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7419	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7420	rc = VINF_SUCCESS;
7421	else
7422	rc = VINF_EM_HALT;
7423
7424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7425	return rc;
7426	}
7427
7428
7429	/**
7430	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7431	*/
7432	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7433	{
7434	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7435	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7436	return VINF_SUCCESS;
7437	}
7438
7439
7440	/**
7441	* VM-exit handler for expiry of the VMX preemption timer.
7442	*/
7443	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7444	{
7445	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7446
7447	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7448	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7449	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7450
7451	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7452	PVM pVM = pVCpu->CTX_SUFF(pVM);
7453	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7455	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7461	*/
7462	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7465	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7466	/** @todo check if XSETBV is supported by the recompiler. */
7467	return VERR_EM_INTERPRETER;
7468	}
7469
7470
7471	/**
7472	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7473	*/
7474	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7475	{
7476	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7477	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7478	/** @todo implement EMInterpretInvpcid() */
7479	return VERR_EM_INTERPRETER;
7480	}
7481
7482
7483	/**
7484	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7485	* Error VM-exit.
7486	*/
7487	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7488	{
7489	uint32_t uIntrState;
7490	RTHCUINTREG uHCReg;
7491	uint64_t u64Val;
7492
7493	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7494	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7495	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7496	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7497	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7498	AssertRCReturn(rc, rc);
7499
7500	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7501	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7502	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7503	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7504
7505	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7506	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7507	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7508	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7509	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7510	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7511	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7512	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7513	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7514	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7515	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7516	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7517
7518	PVM pVM = pVCpu->CTX_SUFF(pVM);
7519	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7520
7521	return VERR_VMX_INVALID_GUEST_STATE;
7522	}
7523
7524
7525	/**
7526	* VM-exit handler for VM-entry failure due to an MSR-load
7527	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7528	*/
7529	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7530	{
7531	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7532	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7533	}
7534
7535
7536	/**
7537	* VM-exit handler for VM-entry failure due to a machine-check event
7538	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7539	*/
7540	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7541	{
7542	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7543	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7544	}
7545
7546
7547	/**
7548	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7549	* theory.
7550	*/
7551	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7552	{
7553	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7554	return VERR_VMX_UNDEFINED_EXIT_CODE;
7555	}
7556
7557
7558	/**
7559	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7560	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7561	* Conditional VM-exit.
7562	*/
7563	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7564	{
7565	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7566	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7568	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7569	return VERR_EM_INTERPRETER;
7570	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7571	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7572	}
7573
7574
7575	/**
7576	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7577	*/
7578	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7579	{
7580	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7581	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7583	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7584	return VERR_EM_INTERPRETER;
7585	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7586	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7587	}
7588
7589
7590	/**
7591	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7592	*/
7593	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7594	{
7595	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7596	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7597	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7598	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7599	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7600	AssertRCReturn(rc, rc);
7601
7602	PVM pVM = pVCpu->CTX_SUFF(pVM);
7603	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7604	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7605	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7606	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7607
7608	/* Update RIP and continue guest execution. */
7609	if (RT_LIKELY(rc == VINF_SUCCESS))
7610	{
7611	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7612	Assert(pVmxTransient->cbInstr == 2);
7613	}
7614	return rc;
7615	}
7616
7617
7618	/**
7619	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7620	*/
7621	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7622	{
7623	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7624	PVM pVM = pVCpu->CTX_SUFF(pVM);
7625	int rc = VINF_SUCCESS;
7626
7627	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7628	if ( pVM->hm.s.fTPRPatchingActive
7629	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7630	{
7631	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7632	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7633	{
7634	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7635	AssertRC(rc);
7636	}
7637
7638	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7639	Assert(pVmxTransient->cbInstr == 2);
7640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7641	return VINF_SUCCESS;
7642	}
7643
7644	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7645	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7646	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7647	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7648	AssertRCReturn(rc, rc);
7649	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7650
7651	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7652	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7654
7655	/* Update guest-state and continue execution. */
7656	if (RT_LIKELY(rc == VINF_SUCCESS))
7657	{
7658	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7659
7660	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7661	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7662	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7663	{
7664	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_APIC_STATE);
7665	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7666	}
7667	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7668	{
7669	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7670	AssertRCReturn(rc, rc);
7671	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7672	}
7673
7674	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7675	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7676	{
7677	switch (pMixedCtx->ecx)
7678	{
7679	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7680	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7681	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7682	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7683	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7684	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7685	}
7686	}
7687	#ifdef VBOX_STRICT
7688	else
7689	{
7690	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7691	switch (pMixedCtx->ecx)
7692	{
7693	case MSR_IA32_SYSENTER_CS:
7694	case MSR_IA32_SYSENTER_EIP:
7695	case MSR_IA32_SYSENTER_ESP:
7696	case MSR_K8_FS_BASE:
7697	case MSR_K8_GS_BASE:
7698	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7699	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7700
7701	case MSR_K8_LSTAR:
7702	case MSR_K6_STAR:
7703	case MSR_K8_SF_MASK:
7704	case MSR_K8_TSC_AUX:
7705	{
7706	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7707	pMixedCtx->ecx));
7708	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7709	}
7710
7711	case MSR_K8_KERNEL_GS_BASE:
7712	{
7713	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7714	pMixedCtx->ecx));
7715	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7716	}
7717	}
7718	}
7719	#endif /* VBOX_STRICT */
7720	}
7721	return rc;
7722	}
7723
7724
7725	/**
7726	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7727	*/
7728	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7729	{
7730	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7731	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7732	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7733	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7734	return VERR_EM_INTERPRETER;
7735	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7736	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7737	}
7738
7739
7740	/**
7741	* VM-exit handler for when the TPR value is lowered below the specified
7742	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7743	*/
7744	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7745	{
7746	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7747	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7748
7749	/*
7750	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7751	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7752	* resume guest execution.
7753	*/
7754	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7755	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7756	return VINF_SUCCESS;
7757	}
7758
7759
7760	/**
7761	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7762	* VM-exit.
7763	*
7764	* @retval VINF_SUCCESS when guest execution can continue.
7765	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7766	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7767	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7768	* recompiler.
7769	*/
7770	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7771	{
7772	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7773	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7774	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7775	AssertRCReturn(rc, rc);
7776
7777	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7778	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7779	PVM pVM = pVCpu->CTX_SUFF(pVM);
7780	switch (uAccessType)
7781	{
7782	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7783	{
7784	#if 0
7785	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7786	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7787	#else
7788	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7789	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7790	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7791	#endif
7792	AssertRCReturn(rc, rc);
7793
7794	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7795	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7796	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7797	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7798
7799	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7800	{
7801	case 0: /* CR0 */
7802	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7803	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7804	break;
7805	case 2: /* C2 **/
7806	/* Nothing to do here, CR2 it's not part of the VMCS. */
7807	break;
7808	case 3: /* CR3 */
7809	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7810	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7811	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7812	break;
7813	case 4: /* CR4 */
7814	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7815	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7816	break;
7817	case 8: /* CR8 */
7818	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7819	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7820	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7821	break;
7822	default:
7823	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7824	break;
7825	}
7826
7827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7828	break;
7829	}
7830
7831	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7832	{
7833	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7834	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7835	AssertRCReturn(rc, rc);
7836	Assert( !pVM->hm.s.fNestedPaging
7837	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7838	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7839
7840	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7841	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7842	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7843
7844	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7845	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7846	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7847	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7848	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7849	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7850	break;
7851	}
7852
7853	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7854	{
7855	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7856	AssertRCReturn(rc, rc);
7857	rc = EMInterpretCLTS(pVM, pVCpu);
7858	AssertRCReturn(rc, rc);
7859	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7861	Log(("CRX CLTS write rc=%d\n", rc));
7862	break;
7863	}
7864
7865	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7866	{
7867	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7868	AssertRCReturn(rc, rc);
7869	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7870	if (RT_LIKELY(rc == VINF_SUCCESS))
7871	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7873	Log(("CRX LMSW write rc=%d\n", rc));
7874	break;
7875	}
7876
7877	default:
7878	{
7879	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7880	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7881	}
7882	}
7883
7884	/* Validate possible error codes. */
7885	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7886	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7887	if (RT_SUCCESS(rc))
7888	{
7889	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7890	AssertRCReturn(rc2, rc2);
7891	}
7892
7893	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7894	return rc;
7895	}
7896
7897
7898	/**
7899	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7900	* VM-exit.
7901	*/
7902	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7903	{
7904	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7905	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7906
7907	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7908	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7909	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7910	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7911	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7912	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7913	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7914	AssertRCReturn(rc, rc);
7915	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7916
7917	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7918	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7919	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7920	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7921	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7922	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7923	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7924
7925	/* I/O operation lookup arrays. */
7926	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7927	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7928
7929	const uint32_t cbSize = s_aIOSize[uIOWidth];
7930	const uint32_t cbInstr = pVmxTransient->cbInstr;
7931	PVM pVM = pVCpu->CTX_SUFF(pVM);
7932	if (fIOString)
7933	{
7934	/* INS/OUTS - I/O String instruction. */
7935	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7936	/** @todo for now manually disassemble later optimize by getting the fields from
7937	* the VMCS. */
7938	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7939	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7940	* segment prefix info. */
7941	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7942	if (RT_SUCCESS(rc))
7943	{
7944	if (fIOWrite)
7945	{
7946	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7947	(DISCPUMODE)pDis->uAddrMode, cbSize);
7948	rc = VBOXSTRICTRC_VAL(rc2);
7949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7950	}
7951	else
7952	{
7953	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7954	(DISCPUMODE)pDis->uAddrMode, cbSize);
7955	rc = VBOXSTRICTRC_VAL(rc2);
7956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7957	}
7958	}
7959	else
7960	{
7961	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7962	rc = VINF_EM_RAW_EMULATE_INSTR;
7963	}
7964	}
7965	else
7966	{
7967	/* IN/OUT - I/O instruction. */
7968	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7969	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7970	if (fIOWrite)
7971	{
7972	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7973	rc = VBOXSTRICTRC_VAL(rc2);
7974	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7975	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7977	}
7978	else
7979	{
7980	uint32_t u32Result = 0;
7981	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7982	rc = VBOXSTRICTRC_VAL(rc2);
7983	if (IOM_SUCCESS(rc))
7984	{
7985	/* Save result of I/O IN instr. in AL/AX/EAX. */
7986	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7987	}
7988	else if (rc == VINF_IOM_R3_IOPORT_READ)
7989	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7991	}
7992	}
7993
7994	if (IOM_SUCCESS(rc))
7995	{
7996	pMixedCtx->rip += cbInstr;
7997	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7998	if (RT_LIKELY(rc == VINF_SUCCESS))
7999	{
8000	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8001	AssertRCReturn(rc, rc);
8002
8003	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8004	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8005	{
8006	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8007	for (unsigned i = 0; i < 4; i++)
8008	{
8009	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8010	if ( ( uIOPort >= pMixedCtx->dr[i]
8011	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8012	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8013	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8014	{
8015	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8016	uint64_t uDR6 = ASMGetDR6();
8017
8018	/* Clear all breakpoint status flags and set the one we just hit. */
8019	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8020	uDR6 \|= (uint64_t)RT_BIT(i);
8021
8022	/*
8023	* Note: AMD64 Architecture Programmer's Manual 13.1:
8024	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8025	* be cleared by software after the contents have been read.
8026	*/
8027	ASMSetDR6(uDR6);
8028
8029	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8030	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8031
8032	/* Paranoia. */
8033	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8034	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8035	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8036
8037	/* Resync DR7 */
8038	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8039	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8040
8041	/* Set #DB to be injected into the VM and continue guest execution. */
8042	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8043	break;
8044	}
8045	}
8046	}
8047	}
8048	}
8049
8050	#ifdef DEBUG
8051	if (rc == VINF_IOM_R3_IOPORT_READ)
8052	Assert(!fIOWrite);
8053	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8054	Assert(fIOWrite);
8055	else
8056	{
8057	AssertMsg( RT_FAILURE(rc)
8058	\|\| rc == VINF_SUCCESS
8059	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8060	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8061	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8062	}
8063	#endif
8064
8065	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8066	return rc;
8067	}
8068
8069
8070	/**
8071	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8072	* VM-exit.
8073	*/
8074	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8075	{
8076	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8077
8078	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8079	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8080	AssertRCReturn(rc, rc);
8081	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8082	{
8083	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8084	AssertRCReturn(rc, rc);
8085	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8086	{
8087	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8088	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8089	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8090	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8091	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8092	{
8093	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8094	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8095
8096	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8097	Assert(!pVCpu->hm.s.Event.fPending);
8098	pVCpu->hm.s.Event.fPending = true;
8099	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8100	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8101	AssertRCReturn(rc, rc);
8102	if (fErrorCodeValid)
8103	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8104	else
8105	pVCpu->hm.s.Event.u32ErrCode = 0;
8106	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8107	&& uVector == X86_XCPT_PF)
8108	{
8109	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8110	}
8111	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8112	}
8113	}
8114	}
8115	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8116	* emulation. */
8117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8118	return VERR_EM_INTERPRETER;
8119	}
8120
8121
8122	/**
8123	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8124	*/
8125	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8126	{
8127	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8128	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8129	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8130	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8131	AssertRCReturn(rc, rc);
8132	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8133	return VINF_EM_DBG_STOP;
8134	}
8135
8136
8137	/**
8138	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8139	*/
8140	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8141	{
8142	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8143
8144	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8145	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8146	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8147	return VINF_SUCCESS;
8148	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8149	return rc;
8150
8151	#if 0
8152	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8153	* just sync the whole thing. */
8154	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8155	#else
8156	/* Aggressive state sync. for now. */
8157	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8158	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8159	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8160	#endif
8161	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8162	AssertRCReturn(rc, rc);
8163
8164	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8165	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8166	switch (uAccessType)
8167	{
8168	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8169	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8170	{
8171	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8172	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8173	{
8174	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8175	}
8176
8177	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8178	GCPhys &= PAGE_BASE_GC_MASK;
8179	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8180	PVM pVM = pVCpu->CTX_SUFF(pVM);
8181	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8182	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8183
8184	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8185	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8186	CPUMCTX2CORE(pMixedCtx), GCPhys);
8187	rc = VBOXSTRICTRC_VAL(rc2);
8188	Log(("ApicAccess rc=%d\n", rc));
8189	if ( rc == VINF_SUCCESS
8190	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8191	\|\| rc == VERR_PAGE_NOT_PRESENT)
8192	{
8193	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8194	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8195	rc = VINF_SUCCESS;
8196	}
8197	break;
8198	}
8199
8200	default:
8201	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8202	rc = VINF_EM_RAW_EMULATE_INSTR;
8203	break;
8204	}
8205
8206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8207	return rc;
8208	}
8209
8210
8211	/**
8212	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8213	* VM-exit.
8214	*/
8215	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8216	{
8217	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8218
8219	/* We should -not- get this VM-exit if the guest is debugging. */
8220	if (CPUMIsGuestDebugStateActive(pVCpu))
8221	{
8222	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8223	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8224	}
8225
8226	int rc = VERR_INTERNAL_ERROR_5;
8227	if ( !DBGFIsStepping(pVCpu)
8228	&& !CPUMIsHyperDebugStateActive(pVCpu))
8229	{
8230	/* Don't intercept MOV DRx. */
8231	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8232	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8233	AssertRCReturn(rc, rc);
8234
8235	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8236	PVM pVM = pVCpu->CTX_SUFF(pVM);
8237	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8238	AssertRC(rc);
8239	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8240
8241	#ifdef VBOX_WITH_STATISTICS
8242	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8243	AssertRCReturn(rc, rc);
8244	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8245	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8246	else
8247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8248	#endif
8249	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8250	return VINF_SUCCESS;
8251	}
8252
8253	/*
8254	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8255	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8256	*/
8257	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8258	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8259	AssertRCReturn(rc, rc);
8260
8261	PVM pVM = pVCpu->CTX_SUFF(pVM);
8262	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8263	{
8264	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8265	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8266	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8267	if (RT_SUCCESS(rc))
8268	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8270	}
8271	else
8272	{
8273	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8274	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8275	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8277	}
8278
8279	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8280	if (RT_SUCCESS(rc))
8281	{
8282	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8283	AssertRCReturn(rc2, rc2);
8284	}
8285	return rc;
8286	}
8287
8288
8289	/**
8290	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8291	* Conditional VM-exit.
8292	*/
8293	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8294	{
8295	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8296	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8297
8298	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8299	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8300	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8301	return VINF_SUCCESS;
8302	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8303	return rc;
8304
8305	RTGCPHYS GCPhys = 0;
8306	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8307
8308	#if 0
8309	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8310	#else
8311	/* Aggressive state sync. for now. */
8312	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8313	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8314	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8315	#endif
8316	AssertRCReturn(rc, rc);
8317
8318	/*
8319	* If we succeed, resume guest execution.
8320	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8321	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8322	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8323	* weird case. See @bugref{6043}.
8324	*/
8325	PVM pVM = pVCpu->CTX_SUFF(pVM);
8326	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8327	rc = VBOXSTRICTRC_VAL(rc2);
8328	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8329	if ( rc == VINF_SUCCESS
8330	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8331	\|\| rc == VERR_PAGE_NOT_PRESENT)
8332	{
8333	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8334	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8335	return VINF_SUCCESS;
8336	}
8337	return rc;
8338	}
8339
8340
8341	/**
8342	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8343	* VM-exit.
8344	*/
8345	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8346	{
8347	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8348	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8349
8350	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8351	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8352	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8353	return VINF_SUCCESS;
8354	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8355	return rc;
8356
8357	RTGCPHYS GCPhys = 0;
8358	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8359	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8360	#if 0
8361	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8362	#else
8363	/* Aggressive state sync. for now. */
8364	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8365	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8366	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8367	#endif
8368	AssertRCReturn(rc, rc);
8369
8370	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8371	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8372
8373	RTGCUINT uErrorCode = 0;
8374	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8375	uErrorCode \|= X86_TRAP_PF_ID;
8376	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8377	uErrorCode \|= X86_TRAP_PF_RW;
8378	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8379	uErrorCode \|= X86_TRAP_PF_P;
8380
8381	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8382
8383	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8384	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8385
8386	/* Handle the pagefault trap for the nested shadow table. */
8387	PVM pVM = pVCpu->CTX_SUFF(pVM);
8388	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8389	TRPMResetTrap(pVCpu);
8390
8391	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8392	if ( rc == VINF_SUCCESS
8393	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8394	\|\| rc == VERR_PAGE_NOT_PRESENT)
8395	{
8396	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8397	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8398	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8399	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8400	return VINF_SUCCESS;
8401	}
8402
8403	Log(("EPT return to ring-3 rc=%d\n"));
8404	return rc;
8405	}
8406
8407
8408	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8409	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8410	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8411	/**
8412	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8413	*/
8414	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8415	{
8416	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8418
8419	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8420	AssertRCReturn(rc, rc);
8421
8422	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8423	{
8424	/* Old-style FPU error reporting needs some extra work. */
8425	/** @todo don't fall back to the recompiler, but do it manually. */
8426	return VERR_EM_INTERPRETER;
8427	}
8428	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8429	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8430	return rc;
8431	}
8432
8433
8434	/**
8435	* VM-exit exception handler for #BP (Breakpoint exception).
8436	*/
8437	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8438	{
8439	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8441
8442	/** @todo Try optimize this by not saving the entire guest state unless
8443	* really needed. */
8444	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8445	AssertRCReturn(rc, rc);
8446
8447	PVM pVM = pVCpu->CTX_SUFF(pVM);
8448	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8449	if (rc == VINF_EM_RAW_GUEST_TRAP)
8450	{
8451	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8452	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8453	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8454	AssertRCReturn(rc, rc);
8455
8456	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8457	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8458	}
8459
8460	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8461	return rc;
8462	}
8463
8464
8465	/**
8466	* VM-exit exception handler for #DB (Debug exception).
8467	*/
8468	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8469	{
8470	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8471	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8472
8473	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8474	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8475	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8476	AssertRCReturn(rc, rc);
8477
8478	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8479	uint64_t uDR6 = X86_DR6_INIT_VAL;
8480	uDR6 \|= (pVmxTransient->uExitQualification
8481	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8482	PVM pVM = pVCpu->CTX_SUFF(pVM);
8483	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8484	if (rc == VINF_EM_RAW_GUEST_TRAP)
8485	{
8486	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8487	pMixedCtx->dr[6] = uDR6;
8488
8489	if (CPUMIsGuestDebugStateActive(pVCpu))
8490	ASMSetDR6(pMixedCtx->dr[6]);
8491
8492	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8493	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8494
8495	/* Paranoia. */
8496	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8497	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8498	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8499
8500	/* Resync DR7. */
8501	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8502
8503	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8504	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8505	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8506	AssertRCReturn(rc,rc);
8507	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8508	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8509	return rc;
8510	}
8511	/* Return to ring 3 to deal with the debug exit code. */
8512	return rc;
8513	}
8514
8515
8516	/**
8517	* VM-exit exception handler for #NM (Device-not-available exception: floating
8518	* point exception).
8519	*/
8520	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8521	{
8522	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8523
8524	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8525	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8526	#endif
8527
8528	/* We require CR0 and EFER. EFER is always up-to-date. */
8529	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8530	AssertRCReturn(rc, rc);
8531
8532	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8533	PVM pVM = pVCpu->CTX_SUFF(pVM);
8534	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8535	if (rc == VINF_SUCCESS)
8536	{
8537	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8538	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8540	return VINF_SUCCESS;
8541	}
8542
8543	/* Forward #NM to the guest. */
8544	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8545	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8546	AssertRCReturn(rc, rc);
8547	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8548	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8550	return rc;
8551	}
8552
8553
8554	/**
8555	* VM-exit exception handler for #GP (General-protection exception).
8556	*
8557	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8558	*/
8559	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8560	{
8561	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8563
8564	int rc = VERR_INTERNAL_ERROR_5;
8565
8566	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8567	{
8568	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8569	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8570	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8571	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8572	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8573	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8574	AssertRCReturn(rc, rc);
8575	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8576	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8577	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8578	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8579	return rc;
8580	#else
8581	/* We don't intercept #GP. */
8582	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8583	return VERR_VMX_UNEXPECTED_EXCEPTION;
8584	#endif
8585	}
8586
8587	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8588	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8589
8590	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8591	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8592	AssertRCReturn(rc, rc);
8593
8594	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8595	uint32_t cbOp = 0;
8596	PVM pVM = pVCpu->CTX_SUFF(pVM);
8597	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8598	if (RT_SUCCESS(rc))
8599	{
8600	rc = VINF_SUCCESS;
8601	Assert(cbOp == pDis->cbInstr);
8602	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8603	switch (pDis->pCurInstr->uOpcode)
8604	{
8605	case OP_CLI:
8606	pMixedCtx->eflags.Bits.u1IF = 0;
8607	pMixedCtx->rip += pDis->cbInstr;
8608	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8610	break;
8611
8612	case OP_STI:
8613	pMixedCtx->eflags.Bits.u1IF = 1;
8614	pMixedCtx->rip += pDis->cbInstr;
8615	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8616	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8617	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8618	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8619	break;
8620
8621	case OP_HLT:
8622	rc = VINF_EM_HALT;
8623	pMixedCtx->rip += pDis->cbInstr;
8624	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8625	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8626	break;
8627
8628	case OP_POPF:
8629	{
8630	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8631	uint32_t cbParm = 0;
8632	uint32_t uMask = 0;
8633	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8634	{
8635	cbParm = 4;
8636	uMask = 0xffffffff;
8637	}
8638	else
8639	{
8640	cbParm = 2;
8641	uMask = 0xffff;
8642	}
8643
8644	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8645	RTGCPTR GCPtrStack = 0;
8646	X86EFLAGS uEflags;
8647	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8648	&GCPtrStack);
8649	if (RT_SUCCESS(rc))
8650	{
8651	Assert(sizeof(uEflags.u32) >= cbParm);
8652	uEflags.u32 = 0;
8653	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8654	}
8655	if (RT_FAILURE(rc))
8656	{
8657	rc = VERR_EM_INTERPRETER;
8658	break;
8659	}
8660	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8661	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8662	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8663	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8664	pMixedCtx->eflags.Bits.u1RF = 0;
8665	pMixedCtx->esp += cbParm;
8666	pMixedCtx->esp &= uMask;
8667	pMixedCtx->rip += pDis->cbInstr;
8668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8670	break;
8671	}
8672
8673	case OP_PUSHF:
8674	{
8675	uint32_t cbParm = 0;
8676	uint32_t uMask = 0;
8677	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8678	{
8679	cbParm = 4;
8680	uMask = 0xffffffff;
8681	}
8682	else
8683	{
8684	cbParm = 2;
8685	uMask = 0xffff;
8686	}
8687
8688	/* Get the stack pointer & push the contents of eflags onto the stack. */
8689	RTGCPTR GCPtrStack = 0;
8690	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8691	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8692	if (RT_FAILURE(rc))
8693	{
8694	rc = VERR_EM_INTERPRETER;
8695	break;
8696	}
8697	X86EFLAGS uEflags;
8698	uEflags = pMixedCtx->eflags;
8699	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8700	uEflags.Bits.u1RF = 0;
8701	uEflags.Bits.u1VM = 0;
8702
8703	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8704	if (RT_FAILURE(rc))
8705	{
8706	rc = VERR_EM_INTERPRETER;
8707	break;
8708	}
8709	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8710	pMixedCtx->esp -= cbParm;
8711	pMixedCtx->esp &= uMask;
8712	pMixedCtx->rip += pDis->cbInstr;
8713	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8714	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8715	break;
8716	}
8717
8718	case OP_IRET:
8719	{
8720	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8721	* instruction reference. */
8722	RTGCPTR GCPtrStack = 0;
8723	uint32_t uMask = 0xffff;
8724	uint16_t aIretFrame[3];
8725	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8726	{
8727	rc = VERR_EM_INTERPRETER;
8728	break;
8729	}
8730	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8731	&GCPtrStack);
8732	if (RT_SUCCESS(rc))
8733	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8734	if (RT_FAILURE(rc))
8735	{
8736	rc = VERR_EM_INTERPRETER;
8737	break;
8738	}
8739	pMixedCtx->eip = 0;
8740	pMixedCtx->ip = aIretFrame[0];
8741	pMixedCtx->cs.Sel = aIretFrame[1];
8742	pMixedCtx->cs.ValidSel = aIretFrame[1];
8743	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8744	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8745	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8746	pMixedCtx->sp += sizeof(aIretFrame);
8747	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8748	\| HM_CHANGED_GUEST_RFLAGS;
8749	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8750	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8751	break;
8752	}
8753
8754	case OP_INT:
8755	{
8756	uint16_t uVector = pDis->Param1.uValue & 0xff;
8757	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8759	break;
8760	}
8761
8762	case OP_INTO:
8763	{
8764	if (pMixedCtx->eflags.Bits.u1OF)
8765	{
8766	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8767	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8768	}
8769	break;
8770	}
8771
8772	default:
8773	{
8774	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8775	EMCODETYPE_SUPERVISOR);
8776	rc = VBOXSTRICTRC_VAL(rc2);
8777	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8778	Log(("#GP rc=%Rrc\n", rc));
8779	break;
8780	}
8781	}
8782	}
8783	else
8784	rc = VERR_EM_INTERPRETER;
8785
8786	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8787	("#GP Unexpected rc=%Rrc\n", rc));
8788	return rc;
8789	}
8790
8791
8792	/**
8793	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8794	* the exception reported in the VMX transient structure back into the VM.
8795	*
8796	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8797	* VMX transient structure to be up-to-date.
8798	*/
8799	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8800	{
8801	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8802
8803	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8804	hmR0VmxCheckExitDueToEventDelivery(). */
8805	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8806	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8807	AssertRCReturn(rc, rc);
8808	Assert(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO);
8809	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8810	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8811	return VINF_SUCCESS;
8812	}
8813
8814
8815	/**
8816	* VM-exit exception handler for #PF (Page-fault exception).
8817	*/
8818	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8819	{
8820	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8821	PVM pVM = pVCpu->CTX_SUFF(pVM);
8822	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8823	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8824	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8825	AssertRCReturn(rc, rc);
8826
8827	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8828	if (pVM->hm.s.fNestedPaging)
8829	{
8830	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8831	{
8832	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8833	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8834	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8835	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8837	}
8838	else
8839	{
8840	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8841	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8842	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8843	Log(("Pending #DF due to vectoring #PF. NP\n"));
8844	}
8845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8846	return rc;
8847	}
8848	#else
8849	Assert(!pVM->hm.s.fNestedPaging);
8850	#endif
8851
8852	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8853	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8854	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8855	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8856	AssertRCReturn(rc, rc);
8857	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8858	if ( pVM->hm.s.fTRPPatchingAllowed
8859	&& pVM->hm.s.pGuestPatchMem
8860	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8861	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8862	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8863	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8864	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8865	{
8866	RTGCPHYS GCPhys;
8867	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8868	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8869	if ( rc == VINF_SUCCESS
8870	&& GCPhys == GCPhysApicBase)
8871	{
8872	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8873	AssertRCReturn(rc, rc);
8874
8875	/* Only attempt to patch the instruction once. */
8876	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8877	if (!pPatch)
8878	return VINF_EM_HM_PATCH_TPR_INSTR;
8879	}
8880	}
8881	#endif
8882
8883	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8884	AssertRCReturn(rc, rc);
8885
8886	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8887	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8888
8889	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8890	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8891	(RTGCPTR)pVmxTransient->uExitQualification);
8892
8893	Log(("#PF: rc=%Rrc\n", rc));
8894	if (rc == VINF_SUCCESS)
8895	{
8896	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8897	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8898	* memory? We don't update the whole state here... */
8899	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8900	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8901	TRPMResetTrap(pVCpu);
8902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8903	return rc;
8904	}
8905	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8906	{
8907	if (!pVmxTransient->fVectoringPF)
8908	{
8909	/* It's a guest page fault and needs to be reflected to the guest. */
8910	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8911	TRPMResetTrap(pVCpu);
8912	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8913	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8914	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8915	0 /* cbInstr */, uGstErrorCode, pMixedCtx->cr2);
8916	}
8917	else
8918	{
8919	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8920	TRPMResetTrap(pVCpu);
8921	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8922	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8923	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8924	}
8925
8926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8927	return VINF_SUCCESS;
8928	}
8929
8930	TRPMResetTrap(pVCpu);
8931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8932	return rc;
8933	}
8934

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45618

Download in other formats: