HMVMXR0.cpp@ 45576

Last change on this file since 45576 was 45576, checked in by vboxsync, 12 years ago
VMMR0/HMVMXR0: oops
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.2 KB

Line
1	/* $Id: HMVMXR0.cpp 45576 2013-04-16 17:02:38Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53	#define VMX_FLUSH_TAGGED_TLB_EPT_VPID 0
54	#define VMX_FLUSH_TAGGED_TLB_EPT 1
55	#define VMX_FLUSH_TAGGED_TLB_VPID 2
56	#define VMX_FLUSH_TAGGED_TLB_NONE 3
57
58	/**
59	* Updated-guest-state flags.
60	*/
61	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
62	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
63	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
64	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
65	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
66	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
67	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
68	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
69	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
70	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
71	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
72	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
73	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
74	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
75	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
76	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
77	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
78	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
79	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
80	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
81	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
82	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
83	\| VMX_UPDATED_GUEST_RIP \
84	\| VMX_UPDATED_GUEST_RSP \
85	\| VMX_UPDATED_GUEST_RFLAGS \
86	\| VMX_UPDATED_GUEST_CR0 \
87	\| VMX_UPDATED_GUEST_CR3 \
88	\| VMX_UPDATED_GUEST_CR4 \
89	\| VMX_UPDATED_GUEST_GDTR \
90	\| VMX_UPDATED_GUEST_IDTR \
91	\| VMX_UPDATED_GUEST_LDTR \
92	\| VMX_UPDATED_GUEST_TR \
93	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
94	\| VMX_UPDATED_GUEST_DEBUG \
95	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
96	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
97	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
98	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
99	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
100	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
101	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
102	\| VMX_UPDATED_GUEST_APIC_STATE)
103
104	/**
105	* Flags to skip redundant reads of some common VMCS fields.
106	*/
107	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
108	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
109	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
110	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
111	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
112	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
113
114	/**
115	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
116	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
117	* we have Nested Paging support.
118	*/
119	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/**
128	* Exception bitmap mask for all contributory exceptions.
129	*/
130	#define VMX_CONTRIBUTORY_XCPT_BITMAP ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
131	\| RT_BIT(X86_XCPT_DE))
132
133	/** Maximum VM-instruction error number. */
134	#define VMX_INSTR_ERROR_MAX 28
135
136	/*******************************************************************************
137	* Structures and Typedefs *
138	*******************************************************************************/
139	/**
140	* A state structure for holding miscellaneous information across
141	* VMX non-root operation and restored after the transition.
142	*/
143	typedef struct VMXTRANSIENT
144	{
145	/** The host's rflags/eflags. */
146	RTCCUINTREG uEFlags;
147	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
148	uint64_t u64LStarMsr;
149	/** The guest's TPR value used for TPR shadowing. */
150	uint8_t u8GuestTpr;
151
152	/** The basic VM-exit reason. */
153	uint16_t uExitReason;
154	/** The VM-exit exit qualification. */
155	RTGCUINTPTR uExitQualification;
156	/** The VM-exit interruption error code. */
157	uint32_t uExitIntrErrorCode;
158
159	/** The VM-exit interruption-information field. */
160	uint32_t uExitIntrInfo;
161	/** Whether the VM-entry failed or not. */
162	bool fVMEntryFailed;
163	/** The VM-exit instruction-length field. */
164	uint32_t cbInstr;
165
166	/** The VM-entry interruption-information field. */
167	uint32_t uEntryIntrInfo;
168	/** The VM-entry exception error code field. */
169	uint32_t uEntryXcptErrorCode;
170	/** The VM-entry instruction length field. */
171	uint32_t cbEntryInstr;
172
173	/** IDT-vectoring information field. */
174	uint32_t uIdtVectoringInfo;
175	/** IDT-vectoring error code. */
176	uint32_t uIdtVectoringErrorCode;
177
178	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
179	uint32_t fVmcsFieldsRead;
180	/** Whether TSC-offsetting should be setup before VM-entry. */
181	bool fUpdateTscOffsettingAndPreemptTimer;
182	/** Whether the VM-exit was caused by a page-fault during delivery of a
183	* contributary exception or a page-fault. */
184	bool fVectoringPF;
185	} VMXTRANSIENT, *PVMXTRANSIENT;
186
187	/**
188	* MSR-bitmap read permissions.
189	*/
190	typedef enum VMXMSREXITREAD
191	{
192	/** Reading this MSR causes a VM-exit. */
193	VMXMSREXIT_INTERCEPT_READ = 0xb,
194	/** Reading this MSR does not cause a VM-exit. */
195	VMXMSREXIT_PASSTHRU_READ
196	} VMXMSREXITREAD;
197
198	/**
199	* MSR-bitmap write permissions.
200	*/
201	typedef enum VMXMSREXITWRITE
202	{
203	/** Writing to this MSR causes a VM-exit. */
204	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
205	/** Writing to this MSR does not cause a VM-exit. */
206	VMXMSREXIT_PASSTHRU_WRITE
207	} VMXMSREXITWRITE;
208
209	/*******************************************************************************
210	* Internal Functions *
211	*******************************************************************************/
212	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
213	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
214	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
215	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
216	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
217	#endif
218	#if 1
219	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
220	#endif
221
222	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273
274	/*******************************************************************************
275	* Global Variables *
276	*******************************************************************************/
277	/**
278	* VM-exit handler.
279	*
280	* @returns VBox status code.
281	* @param pVCpu Pointer to the VMCPU.
282	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
283	* out-of-sync. Make sure to update the required
284	* fields before using them.
285	* @param pVmxTransient Pointer to the VMX-transient structure.
286	*/
287	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	/** Pointer to VM-exit handler. */
289	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
290
291	/**
292	* VMX_EXIT dispatch table.
293	*/
294	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
295	{
296	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
297	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
298	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
299	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
300	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
301	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
302	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
303	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
304	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
305	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
306	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
307	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
308	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
309	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
310	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
311	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
312	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
313	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
314	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
315	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
316	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
317	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
318	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
319	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
320	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
321	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
322	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
323	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
324	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
325	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
326	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
327	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
328	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
329	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
330	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
331	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
333	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
334	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
335	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
336	/* 40 UNDEFINED */ hmR0VmxExitPause,
337	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
338	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
339	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
340	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
341	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
342	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
343	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
344	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
345	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
346	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
347	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
348	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
349	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
350	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
351	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
352	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
353	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
354	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
355	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
356	};
357
358	#ifdef VBOX_STRICT
359	static const char* const g_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
360	{
361	/* 0 */ "(Not Used)",
362	/* 1 */ "VMCALL executed in VMX root operation.",
363	/* 2 */ "VMCLEAR with invalid physical address.",
364	/* 3 */ "VMCLEAR with VMXON pointer.",
365	/* 4 */ "VMLAUNCH with non-clear VMCS.",
366	/* 5 */ "VMRESUME with non-launched VMCS.",
367	/* 6 */ "VMRESUME after VMXOFF",
368	/* 7 */ "VM entry with invalid control fields.",
369	/* 8 */ "VM entry with invalid host state fields.",
370	/* 9 */ "VMPTRLD with invalid physical address.",
371	/* 10 */ "VMPTRLD with VMXON pointer.",
372	/* 11 */ "VMPTRLD with incorrect revision identifier.",
373	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
374	/* 13 */ "VMWRITE to read-only VMCS component.",
375	/* 14 */ "(Not Used)",
376	/* 15 */ "VMXON executed in VMX root operation.",
377	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
378	/* 17 */ "VM entry with non-launched executing VMCS.",
379	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
380	/* 19 */ "VMCALL with non-clear VMCS.",
381	/* 20 */ "VMCALL with invalid VM-exit control fields.",
382	/* 21 */ "(Not Used)",
383	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
384	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
385	/* 24 */ "VMCALL with invalid SMM-monitor features.",
386	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
387	/* 26 */ "VM entry with events blocked by MOV SS.",
388	/* 27 */ "(Not Used)",
389	/* 28 */ "Invalid operand to INVEPT/INVVPID."
390	};
391	#endif
392
393	/**
394	* Updates the VM's last error record. If there was a VMX instruction error,
395	* reads the error data from the VMCS and updates VCPU's last error record as
396	* well.
397	*
398	* @param pVM Pointer to the VM.
399	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
400	* VERR_VMX_UNABLE_TO_START_VM or
401	* VERR_VMX_INVALID_VMCS_FIELD).
402	* @param rc The error code.
403	*/
404	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
405	{
406	AssertPtr(pVM);
407	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
408	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
409	{
410	AssertPtrReturnVoid(pVCpu);
411	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
412	}
413	pVM->hm.s.lLastError = rc;
414	}
415
416
417	/**
418	* Reads the VM-entry interruption-information field from the VMCS into the VMX
419	* transient structure.
420	*
421	* @returns VBox status code.
422	* @param pVmxTransient Pointer to the VMX transient structure.
423	*
424	* @remarks No-long-jump zone!!!
425	*/
426	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
427	{
428	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
429	AssertRCReturn(rc, rc);
430	return VINF_SUCCESS;
431	}
432
433
434	/**
435	* Reads the VM-entry exception error code field from the VMCS into
436	* the VMX transient structure.
437	*
438	* @returns VBox status code.
439	* @param pVmxTransient Pointer to the VMX transient structure.
440	*
441	* @remarks No-long-jump zone!!!
442	*/
443	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
444	{
445	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
446	AssertRCReturn(rc, rc);
447	return VINF_SUCCESS;
448	}
449
450
451	/**
452	* Reads the VM-entry exception error code field from the VMCS into
453	* the VMX transient structure.
454	*
455	* @returns VBox status code.
456	* @param pVCpu Pointer to the VMCPU.
457	* @param pVmxTransient Pointer to the VMX transient structure.
458	*
459	* @remarks No-long-jump zone!!!
460	*/
461	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
462	{
463	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
464	AssertRCReturn(rc, rc);
465	return VINF_SUCCESS;
466	}
467
468
469	/**
470	* Reads the VM-exit interruption-information field from the VMCS into the VMX
471	* transient structure.
472	*
473	* @returns VBox status code.
474	* @param pVCpu Pointer to the VMCPU.
475	* @param pVmxTransient Pointer to the VMX transient structure.
476	*/
477	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
478	{
479	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
480	{
481	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
482	AssertRCReturn(rc, rc);
483	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
484	}
485	return VINF_SUCCESS;
486	}
487
488
489	/**
490	* Reads the VM-exit interruption error code from the VMCS into the VMX
491	* transient structure.
492	*
493	* @returns VBox status code.
494	* @param pVCpu Pointer to the VMCPU.
495	* @param pVmxTransient Pointer to the VMX transient structure.
496	*/
497	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
498	{
499	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
500	{
501	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
502	AssertRCReturn(rc, rc);
503	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
504	}
505	return VINF_SUCCESS;
506	}
507
508
509	/**
510	* Reads the VM-exit instruction length field from the VMCS into the VMX
511	* transient structure.
512	*
513	* @returns VBox status code.
514	* @param pVCpu Pointer to the VMCPU.
515	* @param pVmxTransient Pointer to the VMX transient structure.
516	*/
517	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
518	{
519	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
520	{
521	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
522	AssertRCReturn(rc, rc);
523	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
524	}
525	return VINF_SUCCESS;
526	}
527
528
529	/**
530	* Reads the exit qualification from the VMCS into the VMX transient structure.
531	*
532	* @returns VBox status code.
533	* @param pVCpu Pointer to the VMCPU.
534	* @param pVmxTransient Pointer to the VMX transient structure.
535	*/
536	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
537	{
538	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
539	{
540	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
541	AssertRCReturn(rc, rc);
542	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
543	}
544	return VINF_SUCCESS;
545	}
546
547
548	/**
549	* Reads the IDT-vectoring information field from the VMCS into the VMX
550	* transient structure.
551	*
552	* @returns VBox status code.
553	* @param pVmxTransient Pointer to the VMX transient structure.
554	*
555	* @remarks No-long-jump zone!!!
556	*/
557	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
558	{
559	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
560	{
561	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
562	AssertRCReturn(rc, rc);
563	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
564	}
565	return VINF_SUCCESS;
566	}
567
568
569	/**
570	* Reads the IDT-vectoring error code from the VMCS into the VMX
571	* transient structure.
572	*
573	* @returns VBox status code.
574	* @param pVmxTransient Pointer to the VMX transient structure.
575	*/
576	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577	{
578	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
579	{
580	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
581	AssertRCReturn(rc, rc);
582	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
583	}
584	return VINF_SUCCESS;
585	}
586
587
588	/**
589	* Enters VMX root mode operation on the current CPU.
590	*
591	* @returns VBox status code.
592	* @param pVM Pointer to the VM (optional, can be NULL, after
593	* a resume).
594	* @param HCPhysCpuPage Physical address of the VMXON region.
595	* @param pvCpuPage Pointer to the VMXON region.
596	*/
597	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
598	{
599	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
600	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
601
602	if (pVM)
603	{
604	/* Write the VMCS revision dword to the VMXON region. */
605	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
606	}
607
608	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
609	RTCCUINTREG fFlags = ASMIntDisableFlags();
610
611	/* Enable the VMX bit in CR4 if necessary. */
612	RTCCUINTREG uCr4 = ASMGetCR4();
613	if (!(uCr4 & X86_CR4_VMXE))
614	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
615
616	/* Enter VMX root mode. */
617	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
618	if (RT_FAILURE(rc))
619	ASMSetCR4(uCr4);
620
621	/* Restore interrupts. */
622	ASMSetFlags(fFlags);
623	return rc;
624	}
625
626
627	/**
628	* Exits VMX root mode operation on the current CPU.
629	*
630	* @returns VBox status code.
631	*/
632	static int hmR0VmxLeaveRootMode(void)
633	{
634	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
635	RTCCUINTREG fFlags = ASMIntDisableFlags();
636	int rc = VINF_SUCCESS;
637
638	/* If we're for some reason not in VMX root mode, then don't leave it. */
639	if (ASMGetCR4() & X86_CR4_VMXE)
640	{
641	/* Exit VMX root mode and clear the VMX bit in CR4 */
642	VMXDisable();
643	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
644	}
645	else
646	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
647
648	/* Restore interrupts. */
649	ASMSetFlags(fFlags);
650	return rc;
651	}
652
653
654	/**
655	* Allocates and maps one physically contiguous page. The allocated page is
656	* zero'd out. (Used by various VT-x structures).
657	*
658	* @returns IPRT status code.
659	* @param pMemObj Pointer to the ring-0 memory object.
660	* @param ppVirt Where to store the virtual address of the
661	* allocation.
662	* @param pPhys Where to store the physical address of the
663	* allocation.
664	*/
665	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
666	{
667	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
668	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
669	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
670
671	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
672	if (RT_FAILURE(rc))
673	return rc;
674	ppVirt = RTR0MemObjAddress(pMemObj);
675	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
676	ASMMemZero32(*ppVirt, PAGE_SIZE);
677	return VINF_SUCCESS;
678	}
679
680
681	/**
682	* Frees and unmaps an allocated physical page.
683	*
684	* @param pMemObj Pointer to the ring-0 memory object.
685	* @param ppVirt Where to re-initialize the virtual address of
686	* allocation as 0.
687	* @param pHCPhys Where to re-initialize the physical address of the
688	* allocation as 0.
689	*/
690	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
691	{
692	AssertPtr(pMemObj);
693	AssertPtr(ppVirt);
694	AssertPtr(pHCPhys);
695	if (*pMemObj != NIL_RTR0MEMOBJ)
696	{
697	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
698	AssertRC(rc);
699	*pMemObj = NIL_RTR0MEMOBJ;
700	*ppVirt = 0;
701	*pHCPhys = 0;
702	}
703	}
704
705
706	/**
707	* Worker function to free VT-x related structures.
708	*
709	* @returns IPRT status code.
710	* @param pVM Pointer to the VM.
711	*/
712	static void hmR0VmxStructsFree(PVM pVM)
713	{
714	for (VMCPUID i = 0; i < pVM->cCpus; i++)
715	{
716	PVMCPU pVCpu = &pVM->aCpus[i];
717	AssertPtr(pVCpu);
718
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
720	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
721
722	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
723	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
724
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
726	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
727	}
728
729	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
730	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
731	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
732	#endif
733	}
734
735
736	/**
737	* Worker function to allocate VT-x related VM structures.
738	*
739	* @returns IPRT status code.
740	* @param pVM Pointer to the VM.
741	*/
742	static int hmR0VmxStructsAlloc(PVM pVM)
743	{
744	/*
745	* Initialize members up-front so we can cleanup properly on allocation failure.
746	*/
747	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
748	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
749	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
750	pVM->hm.s.vmx.HCPhys##a_Name = 0;
751
752	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
753	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
754	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
755	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
756
757	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
758	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
759	#endif
760	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
761
762	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
763	for (VMCPUID i = 0; i < pVM->cCpus; i++)
764	{
765	PVMCPU pVCpu = &pVM->aCpus[i];
766	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
767	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
770	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
771	}
772	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
773	#undef VMXLOCAL_INIT_VM_MEMOBJ
774
775	/*
776	* Allocate all the VT-x structures.
777	*/
778	int rc = VINF_SUCCESS;
779	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
780	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
781	if (RT_FAILURE(rc))
782	goto cleanup;
783	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
784	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
785	#endif
786
787	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
788	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
789	{
790	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
791	&pVM->hm.s.vmx.HCPhysApicAccess);
792	if (RT_FAILURE(rc))
793	goto cleanup;
794	}
795
796	/*
797	* Initialize per-VCPU VT-x structures.
798	*/
799	for (VMCPUID i =0; i < pVM->cCpus; i++)
800	{
801	PVMCPU pVCpu = &pVM->aCpus[i];
802	AssertPtr(pVCpu);
803
804	/* Allocate the VM control structure (VMCS). */
805	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
806	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
807	if (RT_FAILURE(rc))
808	goto cleanup;
809
810	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
811	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
812	{
813	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
814	&pVCpu->hm.s.vmx.HCPhysVirtApic);
815	if (RT_FAILURE(rc))
816	goto cleanup;
817	}
818
819	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
820	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
821	{
822	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
823	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
827	}
828
829	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
830	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833
834	/* Allocate the VM-exit MSR-load page for the host MSRs. */
835	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
836	if (RT_FAILURE(rc))
837	goto cleanup;
838	}
839
840	return VINF_SUCCESS;
841
842	cleanup:
843	hmR0VmxStructsFree(pVM);
844	return rc;
845	}
846
847
848	/**
849	* Does global VT-x initialization (called during module initialization).
850	*
851	* @returns VBox status code.
852	*/
853	VMMR0DECL(int) VMXR0GlobalInit(void)
854	{
855	/* Setup the main VM exit handlers. */
856	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
857	#ifdef VBOX_STRICT
858	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
859	Assert(g_apfnVMExitHandlers[i]);
860	#endif
861	return VINF_SUCCESS;
862	}
863
864
865	/**
866	* Does global VT-x termination (called during module termination).
867	*/
868	VMMR0DECL(void) VMXR0GlobalTerm()
869	{
870	/* Nothing to do currently. */
871	}
872
873
874	/**
875	* Sets up and activates VT-x on the current CPU.
876	*
877	* @returns VBox status code.
878	* @param pCpu Pointer to the global CPU info struct.
879	* @param pVM Pointer to the VM (can be NULL after a host resume
880	* operation).
881	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
882	* fEnabledByHost is true).
883	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
884	* @a fEnabledByHost is true).
885	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
886	* enable VT-x/AMD-V on the host.
887	*/
888	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
889	{
890	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
891	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
892
893	if (!fEnabledByHost)
894	{
895	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
896	if (RT_FAILURE(rc))
897	return rc;
898	}
899
900	/*
901	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
902	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
903	* each time while reusing a VPID after hitting the MaxASID limit once.
904	*/
905	if ( pVM
906	&& pVM->hm.s.vmx.fVpid
907	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
908	{
909	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
910	pCpu->fFlushAsidBeforeUse = false;
911	}
912	else
913	pCpu->fFlushAsidBeforeUse = true;
914
915	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
916	++pCpu->cTlbFlushes;
917
918	return VINF_SUCCESS;
919	}
920
921
922	/**
923	* Deactivates VT-x on the current CPU.
924	*
925	* @returns VBox status code.
926	* @param pCpu Pointer to the global CPU info struct.
927	* @param pvCpuPage Pointer to the VMXON region.
928	* @param HCPhysCpuPage Physical address of the VMXON region.
929	*/
930	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
931	{
932	NOREF(pCpu);
933	NOREF(pvCpuPage);
934	NOREF(HCPhysCpuPage);
935
936	hmR0VmxLeaveRootMode();
937	return VINF_SUCCESS;
938	}
939
940
941	/**
942	* Sets the permission bits for the specified MSR in the MSR bitmap.
943	*
944	* @param pVCpu Pointer to the VMCPU.
945	* @param uMSR The MSR value.
946	* @param enmRead Whether reading this MSR causes a VM-exit.
947	* @param enmWrite Whether writing this MSR causes a VM-exit.
948	*/
949	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
950	{
951	int32_t iBit;
952	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
953
954	/*
955	* Layout:
956	* 0x000 - 0x3ff - Low MSR read bits
957	* 0x400 - 0x7ff - High MSR read bits
958	* 0x800 - 0xbff - Low MSR write bits
959	* 0xc00 - 0xfff - High MSR write bits
960	*/
961	if (uMsr <= 0x00001FFF)
962	{
963	/* Pentium-compatible MSRs */
964	iBit = uMsr;
965	}
966	else if ( uMsr >= 0xC0000000
967	&& uMsr <= 0xC0001FFF)
968	{
969	/* AMD Sixth Generation x86 Processor MSRs */
970	iBit = (uMsr - 0xC0000000);
971	pbMsrBitmap += 0x400;
972	}
973	else
974	{
975	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
976	return;
977	}
978
979	Assert(iBit <= 0x1fff);
980	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
981	ASMBitSet(pbMsrBitmap, iBit);
982	else
983	ASMBitClear(pbMsrBitmap, iBit);
984
985	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
986	ASMBitSet(pbMsrBitmap + 0x800, iBit);
987	else
988	ASMBitClear(pbMsrBitmap + 0x800, iBit);
989	}
990
991
992	/**
993	* Flushes the TLB using EPT.
994	*
995	* @returns VBox status code.
996	* @param pVM Pointer to the VM.
997	* @param pVCpu Pointer to the VMCPU.
998	* @param enmFlush Type of flush.
999	*/
1000	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1001	{
1002	AssertPtr(pVM);
1003	Assert(pVM->hm.s.fNestedPaging);
1004
1005	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1006
1007	uint64_t descriptor[2];
1008	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1009	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1010
1011	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1012	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1014	}
1015
1016
1017	/**
1018	* Flushes the TLB using VPID.
1019	*
1020	* @returns VBox status code.
1021	* @param pVM Pointer to the VM.
1022	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1023	* enmFlush).
1024	* @param enmFlush Type of flush.
1025	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1026	* on @a enmFlush).
1027	*/
1028	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1029	{
1030	AssertPtr(pVM);
1031	Assert(pVM->hm.s.vmx.fVpid);
1032
1033	uint64_t descriptor[2];
1034	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1035	{
1036	descriptor[0] = 0;
1037	descriptor[1] = 0;
1038	}
1039	else
1040	{
1041	AssertPtr(pVCpu);
1042	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1043	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1044	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1045	descriptor[1] = GCPtr;
1046	}
1047
1048	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1049	AssertMsg(rc == VINF_SUCCESS,
1050	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1051	if ( RT_SUCCESS(rc)
1052	&& pVCpu)
1053	{
1054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1055	}
1056	}
1057
1058
1059	/**
1060	* Invalidates a guest page by guest virtual address. Only relevant for
1061	* EPT/VPID, otherwise there is nothing really to invalidate.
1062	*
1063	* @returns VBox status code.
1064	* @param pVM Pointer to the VM.
1065	* @param pVCpu Pointer to the VMCPU.
1066	* @param GCVirt Guest virtual address of the page to invalidate.
1067	*/
1068	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1069	{
1070	AssertPtr(pVM);
1071	AssertPtr(pVCpu);
1072	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1073
1074	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1075	if (!fFlushPending)
1076	{
1077	/*
1078	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1079	* See @bugref{6043} and @bugref{6177}.
1080	*
1081	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1082	* function maybe called in a loop with individual addresses.
1083	*/
1084	if (pVM->hm.s.vmx.fVpid)
1085	{
1086	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1087	{
1088	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1089	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1090	}
1091	else
1092	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1093	}
1094	else if (pVM->hm.s.fNestedPaging)
1095	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1096	}
1097
1098	return VINF_SUCCESS;
1099	}
1100
1101
1102	/**
1103	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1104	* otherwise there is nothing really to invalidate.
1105	*
1106	* @returns VBox status code.
1107	* @param pVM Pointer to the VM.
1108	* @param pVCpu Pointer to the VMCPU.
1109	* @param GCPhys Guest physical address of the page to invalidate.
1110	*/
1111	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1112	{
1113	LogFlowFunc(("%RGp\n", GCPhys));
1114
1115	/*
1116	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1117	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1118	* This function might be called in a loop.
1119	*/
1120	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1121	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1128	* case where neither EPT nor VPID is supported by the CPU.
1129	*
1130	* @param pVM Pointer to the VM.
1131	* @param pVCpu Pointer to the VMCPU.
1132	*
1133	* @remarks Called with interrupts disabled.
1134	*/
1135	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1136	{
1137	NOREF(pVM);
1138	AssertPtr(pVCpu);
1139	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1140	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1141
1142	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1143	AssertPtr(pCpu);
1144
1145	pVCpu->hm.s.TlbShootdown.cPages = 0;
1146	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1147	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1148	pVCpu->hm.s.fForceTLBFlush = false;
1149	return;
1150	}
1151
1152
1153	/**
1154	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1155	*
1156	* @param pVM Pointer to the VM.
1157	* @param pVCpu Pointer to the VMCPU.
1158	* @remarks All references to "ASID" in this function pertains to "VPID" in
1159	* Intel's nomenclature. The reason is, to avoid confusion in compare
1160	* statements since the host-CPU copies are named "ASID".
1161	*
1162	* @remarks Called with interrupts disabled.
1163	*/
1164	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1165	{
1166	AssertPtr(pVM);
1167	AssertPtr(pVCpu);
1168	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1169	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1170	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1171
1172	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1173	AssertPtr(pCpu);
1174
1175	/*
1176	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1177	* This can happen both for start & resume due to long jumps back to ring-3.
1178	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1179	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1180	*/
1181	bool fNewASID = false;
1182	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1183	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1184	{
1185	pVCpu->hm.s.fForceTLBFlush = true;
1186	fNewASID = true;
1187	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1188	}
1189
1190	/*
1191	* Check for explicit TLB shootdowns.
1192	*/
1193	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1194	{
1195	pVCpu->hm.s.fForceTLBFlush = true;
1196	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1197	}
1198
1199	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1200	if (pVCpu->hm.s.fForceTLBFlush)
1201	{
1202	if (fNewASID)
1203	{
1204	++pCpu->uCurrentAsid;
1205	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1206	{
1207	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1208	pCpu->cTlbFlushes++;
1209	pCpu->fFlushAsidBeforeUse = true;
1210	}
1211
1212	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1213	if (pCpu->fFlushAsidBeforeUse)
1214	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1215	}
1216	else
1217	{
1218	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1219	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1220	else
1221	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1222	}
1223
1224	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1225	pVCpu->hm.s.fForceTLBFlush = false;
1226	}
1227	else
1228	{
1229	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1230	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1231	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1232	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1233
1234	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1235	* not be executed. See hmQueueInvlPage() where it is commented
1236	* out. Support individual entry flushing someday. */
1237	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1238	{
1239	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1240
1241	/*
1242	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1243	* as supported by the CPU.
1244	*/
1245	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1246	{
1247	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1248	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1249	}
1250	else
1251	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1252	}
1253	else
1254	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1255	}
1256	pVCpu->hm.s.TlbShootdown.cPages = 0;
1257	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1258
1259	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1260	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1261	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1262	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1263	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1264	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1265
1266	/* Update VMCS with the VPID. */
1267	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1268	AssertRC(rc);
1269	}
1270
1271
1272	/**
1273	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1274	*
1275	* @returns VBox status code.
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	*
1279	* @remarks Called with interrupts disabled.
1280	*/
1281	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1282	{
1283	AssertPtr(pVM);
1284	AssertPtr(pVCpu);
1285	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1286	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1287
1288	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1289	AssertPtr(pCpu);
1290
1291	/*
1292	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1293	* This can happen both for start & resume due to long jumps back to ring-3.
1294	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1295	*/
1296	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1297	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1298	{
1299	pVCpu->hm.s.fForceTLBFlush = true;
1300	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1301	}
1302
1303	/* Check for explicit TLB shootdown flushes. */
1304	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1305	{
1306	pVCpu->hm.s.fForceTLBFlush = true;
1307	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1308	}
1309
1310	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1311	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1312
1313	if (pVCpu->hm.s.fForceTLBFlush)
1314	{
1315	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1316	pVCpu->hm.s.fForceTLBFlush = false;
1317	}
1318	else
1319	{
1320	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1321	* not be executed. See hmQueueInvlPage() where it is commented
1322	* out. Support individual entry flushing someday. */
1323	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1324	{
1325	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1326	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1327	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1328	}
1329	else
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1331	}
1332
1333	pVCpu->hm.s.TlbShootdown.cPages = 0;
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1335	}
1336
1337
1338	/**
1339	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1340	*
1341	* @returns VBox status code.
1342	* @param pVM Pointer to the VM.
1343	* @param pVCpu Pointer to the VMCPU.
1344	*
1345	* @remarks Called with interrupts disabled.
1346	*/
1347	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1348	{
1349	AssertPtr(pVM);
1350	AssertPtr(pVCpu);
1351	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1352	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1353
1354	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1355
1356	/*
1357	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1358	* This can happen both for start & resume due to long jumps back to ring-3.
1359	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1360	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1361	*/
1362	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1363	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1364	{
1365	pVCpu->hm.s.fForceTLBFlush = true;
1366	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1367	}
1368
1369	/* Check for explicit TLB shootdown flushes. */
1370	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1371	{
1372	/*
1373	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1374	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1375	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1376	*/
1377	pVCpu->hm.s.fForceTLBFlush = true;
1378	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1379	}
1380
1381	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1382	if (pVCpu->hm.s.fForceTLBFlush)
1383	{
1384	++pCpu->uCurrentAsid;
1385	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1386	{
1387	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1388	pCpu->fFlushAsidBeforeUse = true;
1389	pCpu->cTlbFlushes++;
1390	}
1391
1392	pVCpu->hm.s.fForceTLBFlush = false;
1393	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1394	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1395	if (pCpu->fFlushAsidBeforeUse)
1396	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1397	}
1398	else
1399	{
1400	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1401	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1402	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1403	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1404
1405	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1406	* not be executed. See hmQueueInvlPage() where it is commented
1407	* out. Support individual entry flushing someday. */
1408	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1409	{
1410	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1411	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1412	{
1413	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1414	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1415	}
1416	else
1417	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1418	}
1419	else
1420	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1421	}
1422
1423	pVCpu->hm.s.TlbShootdown.cPages = 0;
1424	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1425
1426	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1427	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1428	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1429	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1430	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1431	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1432
1433	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1434	AssertRC(rc);
1435	}
1436
1437
1438	/**
1439	* Flushes the guest TLB entry based on CPU capabilities.
1440	*
1441	* @param pVCpu Pointer to the VMCPU.
1442	*/
1443	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1444	{
1445	PVM pVM = pVCpu->CTX_SUFF(pVM);
1446	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1447	{
1448	case VMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1449	case VMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1450	case VMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1451	case VMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1452	default:
1453	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1454	break;
1455	}
1456	}
1457
1458
1459	/**
1460	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1461	* TLB entries from the host TLB before VM-entry.
1462	*
1463	* @returns VBox status code.
1464	* @param pVM Pointer to the VM.
1465	*/
1466	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1467	{
1468	/*
1469	* Determine optimal flush type for nested paging.
1470	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1471	* guest execution (see hmR3InitFinalizeR0()).
1472	*/
1473	if (pVM->hm.s.fNestedPaging)
1474	{
1475	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1478	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1479	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1480	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1481	else
1482	{
1483	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1484	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1485	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1486	}
1487
1488	/* Make sure the write-back cacheable memory type for EPT is supported. */
1489	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1490	{
1491	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1492	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1493	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1494	}
1495	}
1496	else
1497	{
1498	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1499	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1500	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1501	}
1502	}
1503
1504	/*
1505	* Determine optimal flush type for VPID.
1506	*/
1507	if (pVM->hm.s.vmx.fVpid)
1508	{
1509	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1510	{
1511	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1512	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1513	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1515	else
1516	{
1517	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1518	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1519	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1520	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1521	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1522	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1523	pVM->hm.s.vmx.fVpid = false;
1524	}
1525	}
1526	else
1527	{
1528	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1529	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1530	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1531	pVM->hm.s.vmx.fVpid = false;
1532	}
1533	}
1534
1535	/*
1536	* Setup the handler for flushing tagged-TLBs.
1537	*/
1538	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1539	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT_VPID;
1540	else if (pVM->hm.s.fNestedPaging)
1541	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT;
1542	else if (pVM->hm.s.vmx.fVpid)
1543	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_VPID;
1544	else
1545	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_NONE;
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/**
1551	* Sets up pin-based VM-execution controls in the VMCS.
1552	*
1553	* @returns VBox status code.
1554	* @param pVM Pointer to the VM.
1555	* @param pVCpu Pointer to the VMCPU.
1556	*/
1557	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1558	{
1559	AssertPtr(pVM);
1560	AssertPtr(pVCpu);
1561
1562	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1563	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1564
1565	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1566	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1567	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1568
1569	/* Enable the VMX preemption timer. */
1570	if (pVM->hm.s.vmx.fUsePreemptTimer)
1571	{
1572	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1573	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1574	}
1575
1576	if ((val & zap) != val)
1577	{
1578	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1579	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1580	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1581	}
1582
1583	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1584	AssertRCReturn(rc, rc);
1585
1586	/* Update VCPU with the currently set pin-based VM-execution controls. */
1587	pVCpu->hm.s.vmx.u32PinCtls = val;
1588	return rc;
1589	}
1590
1591
1592	/**
1593	* Sets up processor-based VM-execution controls in the VMCS.
1594	*
1595	* @returns VBox status code.
1596	* @param pVM Pointer to the VM.
1597	* @param pVMCPU Pointer to the VMCPU.
1598	*/
1599	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1600	{
1601	AssertPtr(pVM);
1602	AssertPtr(pVCpu);
1603
1604	int rc = VERR_INTERNAL_ERROR_5;
1605	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1606	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1607
1608	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1609	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1610	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1613	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1614	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1615
1616	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1617	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1618	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1619	{
1620	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1621	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1622	}
1623
1624	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1625	if (!pVM->hm.s.fNestedPaging)
1626	{
1627	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1628	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1629	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1630	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1631	}
1632
1633	/* Use TPR shadowing if supported by the CPU. */
1634	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1635	{
1636	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1637	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1638	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1639	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1640	AssertRCReturn(rc, rc);
1641
1642	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1643	/* CR8 writes causes a VM-exit based on TPR threshold. */
1644	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1645	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1646	}
1647	else
1648	{
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1651	}
1652
1653	/* Use MSR-bitmaps if supported by the CPU. */
1654	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1655	{
1656	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1657
1658	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1659	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1660	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1661	AssertRCReturn(rc, rc);
1662
1663	/*
1664	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1665	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1666	*/
1667	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1668	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1669	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1670	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1671	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1672	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1673	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1674	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1675	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1676	}
1677
1678	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1679	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1680	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1681
1682	if ((val & zap) != val)
1683	{
1684	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1685	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1687	}
1688
1689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1690	AssertRCReturn(rc, rc);
1691
1692	/* Update VCPU with the currently set processor-based VM-execution controls. */
1693	pVCpu->hm.s.vmx.u32ProcCtls = val;
1694
1695	/*
1696	* Secondary processor-based VM-execution controls.
1697	*/
1698	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1699	{
1700	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1701	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1702
1703	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1704	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1705
1706	if (pVM->hm.s.fNestedPaging)
1707	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1708	else
1709	{
1710	/*
1711	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1712	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1713	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1714	*/
1715	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1716	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1717	}
1718
1719	if (pVM->hm.s.vmx.fVpid)
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1721
1722	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1723	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1724
1725	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1726	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1727	* done dynamically. */
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1729	{
1730	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1731	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1732	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1733	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1734	AssertRCReturn(rc, rc);
1735	}
1736
1737	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1738	{
1739	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1740	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	}
1743
1744	if ((val & zap) != val)
1745	{
1746	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1747	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1748	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1749	}
1750
1751	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1752	AssertRCReturn(rc, rc);
1753
1754	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1755	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1756	}
1757
1758	return VINF_SUCCESS;
1759	}
1760
1761
1762	/**
1763	* Sets up miscellaneous (everything other than Pin & Processor-based
1764	* VM-execution) control fields in the VMCS.
1765	*
1766	* @returns VBox status code.
1767	* @param pVM Pointer to the VM.
1768	* @param pVCpu Pointer to the VMCPU.
1769	*/
1770	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1771	{
1772	AssertPtr(pVM);
1773	AssertPtr(pVCpu);
1774
1775	int rc = VERR_GENERAL_FAILURE;
1776
1777	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1778	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1779
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1781
1782	/*
1783	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1784	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1785	* We thus use the exception bitmap to control it rather than use both.
1786	*/
1787	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1788	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1789
1790	/** @todo Explore possibility of using IO-bitmaps. */
1791	/* All IO & IOIO instructions cause VM-exits. */
1792	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1793	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1794
1795	/* Setup MSR autoloading/autostoring. */
1796	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1797	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1798	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1799	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1800	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1801	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1802
1803	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1804	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1805	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1806	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1807
1808	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1809	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1810
1811	/* Setup debug controls */
1812	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1813	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1814	AssertRCReturn(rc, rc);
1815	return rc;
1816	}
1817
1818
1819	/**
1820	* Sets up the initial exception bitmap in the VMCS based on static conditions
1821	* (i.e. conditions that cannot ever change at runtime).
1822	*
1823	* @returns VBox status code.
1824	* @param pVM Pointer to the VM.
1825	* @param pVCpu Pointer to the VMCPU.
1826	*/
1827	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1828	{
1829	AssertPtr(pVM);
1830	AssertPtr(pVCpu);
1831
1832	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1833
1834	uint32_t u32XcptBitmap = 0;
1835
1836	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1837	if (!pVM->hm.s.fNestedPaging)
1838	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1839
1840	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1841	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1842	AssertRCReturn(rc, rc);
1843	return rc;
1844	}
1845
1846
1847	/**
1848	* Sets up the initial guest-state mask. The guest-state mask is consulted
1849	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1850	* for the nested virtualization case (as it would cause a VM-exit).
1851	*
1852	* @param pVCpu Pointer to the VMCPU.
1853	*/
1854	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1855	{
1856	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1857	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1858	return VINF_SUCCESS;
1859	}
1860
1861
1862	/**
1863	* Does per-VM VT-x initialization.
1864	*
1865	* @returns VBox status code.
1866	* @param pVM Pointer to the VM.
1867	*/
1868	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1869	{
1870	LogFlowFunc(("pVM=%p\n", pVM));
1871
1872	int rc = hmR0VmxStructsAlloc(pVM);
1873	if (RT_FAILURE(rc))
1874	{
1875	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1876	return rc;
1877	}
1878
1879	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1880	{
1881	PVMCPU pVCpu = &pVM->aCpus[i];
1882
1883	/* Current guest paging mode. */
1884	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Does per-VM VT-x termination.
1893	*
1894	* @returns VBox status code.
1895	* @param pVM Pointer to the VM.
1896	*/
1897	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1898	{
1899	LogFlowFunc(("pVM=%p\n", pVM));
1900
1901	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1902	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1903	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1904	#endif
1905	hmR0VmxStructsFree(pVM);
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Sets up the VM for execution under VT-x.
1912	* This function is only called once per-VM during initalization.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	*/
1917	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1918	{
1919	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1920	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1921
1922	LogFlowFunc(("pVM=%p\n", pVM));
1923
1924	/*
1925	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1926	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1927	*/
1928	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1929	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1930	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1931	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1932	{
1933	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1934	return VERR_INTERNAL_ERROR;
1935	}
1936
1937	/* Initialize these always, see hmR3InitFinalizeR0().*/
1938	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1939	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1940
1941	/* Setup the tagged-TLB flush handlers. */
1942	int rc = hmR0VmxSetupTaggedTlb(pVM);
1943	if (RT_FAILURE(rc))
1944	{
1945	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1946	return rc;
1947	}
1948
1949	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1950	{
1951	PVMCPU pVCpu = &pVM->aCpus[i];
1952	AssertPtr(pVCpu);
1953	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1954
1955	/* Set revision dword at the beginning of the VMCS structure. */
1956	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1957
1958	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1959	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1960	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1961	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1962
1963	/* Load this VMCS as the current VMCS. */
1964	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1965	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1966	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1967
1968	/* Setup the pin-based VM-execution controls. */
1969	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1970	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1971	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1972
1973	/* Setup the processor-based VM-execution controls. */
1974	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1975	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1976	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1977
1978	/* Setup the rest (miscellaneous) VM-execution controls. */
1979	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1980	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1981	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1982
1983	/* Setup the initial exception bitmap. */
1984	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1985	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1986	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1987
1988	/* Setup the initial guest-state mask. */
1989	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1990	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1991	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1992
1993	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1994	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1995	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1996	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1997	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1998	#endif
1999
2000	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2001	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2002	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2003	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2004
2005	/* Update the last error record for this VCPU. */
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2007	}
2008
2009	return VINF_SUCCESS;
2010	}
2011
2012
2013	/**
2014	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2015	* the VMCS.
2016	*
2017	* @returns VBox status code.
2018	* @param pVM Pointer to the VM.
2019	* @param pVCpu Pointer to the VMCPU.
2020	*/
2021	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2022	{
2023	RTCCUINTREG uReg = ASMGetCR0();
2024	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2025
2026	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2027	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2028	if (VMX_IS_64BIT_HOST_MODE())
2029	{
2030	uint64_t uReg = hmR0Get64bitCR3();
2031	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2032	}
2033	else
2034	#endif
2035	{
2036	uReg = ASMGetCR3();
2037	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2038	}
2039
2040	uReg = ASMGetCR4();
2041	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2042	AssertRCReturn(rc, rc);
2043	return rc;
2044	}
2045
2046
2047	/**
2048	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2049	* the host-state area in the VMCS.
2050	*
2051	* @returns VBox status code.
2052	* @param pVM Pointer to the VM.
2053	* @param pVCpu Pointer to the VMCPU.
2054	*/
2055	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2056	{
2057	int rc = VERR_INTERNAL_ERROR_5;
2058	RTSEL uSelCS = 0;
2059	RTSEL uSelSS = 0;
2060	RTSEL uSelDS = 0;
2061	RTSEL uSelES = 0;
2062	RTSEL uSelFS = 0;
2063	RTSEL uSelGS = 0;
2064	RTSEL uSelTR = 0;
2065
2066	/*
2067	* Host Selector registers.
2068	*/
2069	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2070	if (VMX_IS_64BIT_HOST_MODE())
2071	{
2072	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2073	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2074	}
2075	else
2076	{
2077	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2078	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2079	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2080	}
2081	#else
2082	uSelCS = ASMGetCS();
2083	uSelSS = ASMGetSS();
2084	#endif
2085
2086	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2087	/** @todo Verify if we have any platform that actually run with DS or ES with
2088	* RPL != 0 in kernel space. */
2089	uSelDS = 0;
2090	uSelES = 0;
2091	uSelFS = 0;
2092	uSelGS = 0;
2093	uSelTR = ASMGetTR();
2094
2095	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2096	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2097	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2098	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2099	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2100	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2101	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2102	Assert(uSelCS != 0);
2103	Assert(uSelTR != 0);
2104
2105	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2106	#if 0
2107	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2108	Assert(uSelSS != 0);
2109	#endif
2110
2111	/* Write these host selector fields into the host-state area in the VMCS. */
2112	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2113	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2114	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2115	#if 0
2116	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2117	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2118	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2119	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2120	#endif
2121	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2122	AssertRCReturn(rc, rc);
2123
2124	/*
2125	* Host GDTR and IDTR.
2126	*/
2127	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2128	* be safe to -not- save and restore GDTR and IDTR in the assembly
2129	* code and just do it here and don't care if the limits are zapped on
2130	* VM-exit. */
2131	RTGDTR Gdtr;
2132	RT_ZERO(Gdtr);
2133	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2134	if (VMX_IS_64BIT_HOST_MODE())
2135	{
2136	X86XDTR64 Gtr64;
2137	X86XDTR64 Idtr64;
2138	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2139	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2140	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2141	Gdtr.cbGdt = Gdtr64.cb;
2142	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2143	}
2144	else
2145	#endif
2146	{
2147	RTIDTR Idtr;
2148	ASMGetGDTR(&Gdtr);
2149	ASMGetIDTR(&Idtr);
2150	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2151	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2152	}
2153	AssertRCReturn(rc, rc);
2154
2155	/*
2156	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2157	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2158	*/
2159	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2160	{
2161	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2162	return VERR_VMX_INVALID_HOST_STATE;
2163	}
2164
2165	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2166	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2167	if (VMX_IS_64BIT_HOST_MODE())
2168	{
2169	/* We need the 64-bit TR base for hybrid darwin. */
2170	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2171	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2172	}
2173	else
2174	#endif
2175	{
2176	uintptr_t uTRBase;
2177	#if HC_ARCH_BITS == 64
2178	uTRBase = X86DESC64_BASE(pDesc);
2179	#else
2180	uTRBase = X86DESC_BASE(pDesc);
2181	#endif
2182	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2183	}
2184	AssertRCReturn(rc, rc);
2185
2186	/*
2187	* Host FS base and GS base.
2188	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2189	* would take care of the bases. In 64-bit, the MSRs come into play.
2190	*/
2191	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2192	if (VMX_IS_64BIT_HOST_MODE())
2193	{
2194	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2195	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2196	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2197	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2198	AssertRCReturn(rc, rc);
2199	}
2200	#endif
2201	return rc;
2202	}
2203
2204
2205	/**
2206	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2207	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2208	* the host after every successful VM exit.
2209	*
2210	* @returns VBox status code.
2211	* @param pVM Pointer to the VM.
2212	* @param pVCpu Pointer to the VMCPU.
2213	*/
2214	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2215	{
2216	AssertPtr(pVCpu);
2217	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2218
2219	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2220	uint32_t cHostMsrs = 0;
2221	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2222
2223	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2224	{
2225	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2226	pHostMsr->u32Reserved = 0;
2227	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2228	if (CPUMIsGuestInLongMode(pVCpu))
2229	{
2230	/* Must match the EFER value in our 64 bits switcher. */
2231	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2232	}
2233	else
2234	#endif
2235	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2236	pHostMsr++; cHostMsrs++;
2237	}
2238
2239	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2240	if (VMX_IS_64BIT_HOST_MODE())
2241	{
2242	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2243	pHostMsr->u32Reserved = 0;
2244	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2245	pHostMsr++; cHostMsrs++;
2246	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2247	pHostMsr->u32Reserved = 0;
2248	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2249	pHostMsr++; cHostMsrs++;
2250	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2251	pHostMsr->u32Reserved = 0;
2252	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2253	pHostMsr++; cHostMsrs++;
2254	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2255	#if 0
2256	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2257	pMsr->u32Reserved = 0;
2258	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2259	pHostMsr++; cHostMsrs++;
2260	#endif
2261	}
2262	#endif
2263
2264	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2265	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2266	{
2267	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2268	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269	}
2270
2271	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2272
2273	/*
2274	* Host Sysenter MSRs.
2275	*/
2276	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2277	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2278	if (VMX_IS_64BIT_HOST_MODE())
2279	{
2280	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2281	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2282	}
2283	else
2284	{
2285	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2286	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2287	}
2288	#elif HC_ARCH_BITS == 32
2289	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2290	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2291	#else
2292	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2293	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2294	#endif
2295	AssertRCReturn(rc, rc);
2296
2297	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2298	* hmR0VmxSetupExitCtls() !! */
2299	return rc;
2300	}
2301
2302
2303	/**
2304	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2305	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2306	* controls".
2307	*
2308	* @returns VBox status code.
2309	* @param pVM Pointer to the VM.
2310	* @param pVCpu Pointer to the VMCPU.
2311	* @param pCtx Pointer to the guest-CPU context.
2312	*
2313	* @remarks No-long-jump zone!!!
2314	*/
2315	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2316	{
2317	int rc = VINF_SUCCESS;
2318	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2319	{
2320	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2321	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2322
2323	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2324	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2325
2326	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2327	if (CPUMIsGuestInLongModeEx(pCtx))
2328	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2329	else
2330	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2331
2332	/*
2333	* The following should not be set (since we're not in SMM mode):
2334	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2335	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2336	*/
2337
2338	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2339	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2340	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2341
2342	if ((val & zap) != val)
2343	{
2344	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2345	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2346	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2347	}
2348
2349	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2350	AssertRCReturn(rc, rc);
2351
2352	/* Update VCPU with the currently set VM-exit controls. */
2353	pVCpu->hm.s.vmx.u32EntryCtls = val;
2354	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2355	}
2356	return rc;
2357	}
2358
2359
2360	/**
2361	* Sets up the VM-exit controls in the VMCS.
2362	*
2363	* @returns VBox status code.
2364	* @param pVM Pointer to the VM.
2365	* @param pVCpu Pointer to the VMCPU.
2366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2367	* out-of-sync. Make sure to update the required fields
2368	* before using them.
2369	*
2370	* @remarks requires EFER.
2371	*/
2372	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2373	{
2374	int rc = VINF_SUCCESS;
2375	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2376	{
2377	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2378	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2379
2380	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2381	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2382
2383	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2384	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2385	if (VMX_IS_64BIT_HOST_MODE())
2386	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2387	else
2388	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2389	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2390	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2391	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2392	else
2393	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2394	#endif
2395
2396	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2397	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2398
2399	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2400	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2401	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2402	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2403	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2404
2405	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2406	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2411	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2412	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2413	}
2414
2415	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2416	AssertRCReturn(rc, rc);
2417
2418	/* Update VCPU with the currently set VM-exit controls. */
2419	pVCpu->hm.s.vmx.u32ExitCtls = val;
2420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2421	}
2422	return rc;
2423	}
2424
2425
2426	/**
2427	* Loads the guest APIC and related state.
2428	*
2429	* @returns VBox status code.
2430	* @param pVM Pointer to the VM.
2431	* @param pVCpu Pointer to the VMCPU.
2432	* @param pCtx Pointer to the guest-CPU context.
2433	*/
2434	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2435	{
2436	int rc = VINF_SUCCESS;
2437	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2438	{
2439	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2440	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2441	{
2442	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2443
2444	bool fPendingIntr = false;
2445	uint8_t u8GuestTpr = 0;
2446	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2447	AssertRCReturn(rc, rc);
2448
2449	/*
2450	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2451	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2452	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2453	* the interrupt when we VM-exit for other reasons.
2454	*/
2455	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2456	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2457	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2458	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2459
2460	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2461	AssertRCReturn(rc, rc);
2462
2463	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2464	if (pVM->hm.s.fTPRPatchingActive)
2465	{
2466	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2467	pCtx->msrLSTAR = u8GuestTpr;
2468	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2469	{
2470	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2471	if (fPendingIntr)
2472	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2473	else
2474	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2475	}
2476	}
2477	}
2478
2479	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2480	}
2481	return rc;
2482	}
2483
2484
2485	/**
2486	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2487	*
2488	* @returns
2489	* @param pVCpu Pointer to the VMCPU.
2490	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2491	* out-of-sync. Make sure to update the required fields
2492	* before using them.
2493	*
2494	* @remarks No-long-jump zone!!!
2495	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2496	*/
2497	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2498	{
2499	/*
2500	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2501	* inhibit interrupts or clear any existing interrupt-inhibition.
2502	*/
2503	uint32_t uIntrState = 0;
2504	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2505	{
2506	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2507	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS))
2508	== (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2509	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2510	{
2511	/*
2512	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2513	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2514	*/
2515	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2516	}
2517	else if (pMixedCtx->eflags.Bits.u1IF)
2518	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2519	else
2520	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2521	}
2522	return uIntrState;
2523	}
2524
2525
2526	/**
2527	* Loads the guest's interruptibility-state into the guest-state area in the
2528	* VMCS.
2529	*
2530	* @returns VBox status code.
2531	* @param pVCpu Pointer to the VMCPU.
2532	* @param uIntrState The interruptibility-state to set.
2533	*/
2534	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2535	{
2536	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2537	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2538	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2539	AssertRCReturn(rc, rc);
2540	return rc;
2541	}
2542
2543
2544	/**
2545	* Loads the guest's RIP into the guest-state area in the VMCS.
2546	*
2547	* @returns VBox status code.
2548	* @param pVM Pointer to the VM.
2549	* @param pVCpu Pointer to the VMCPU.
2550	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2551	* out-of-sync. Make sure to update the required fields
2552	* before using them.
2553	*
2554	* @remarks No-long-jump zone!!!
2555	*/
2556	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2557	{
2558	int rc = VINF_SUCCESS;
2559	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2560	{
2561	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2562	AssertRCReturn(rc, rc);
2563	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2564	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2565	}
2566	return rc;
2567	}
2568
2569
2570	/**
2571	* Loads the guest's RSP into the guest-state area in the VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @param pVM Pointer to the VM.
2575	* @param pVCpu Pointer to the VMCPU.
2576	* @param pCtx Pointer to the guest-CPU context.
2577	*
2578	* @remarks No-long-jump zone!!!
2579	*/
2580	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2581	{
2582	int rc = VINF_SUCCESS;
2583	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2584	{
2585	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2586	AssertRCReturn(rc, rc);
2587	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2588	}
2589	return rc;
2590	}
2591
2592
2593	/**
2594	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2595	*
2596	* @returns VBox status code.
2597	* @param pVM Pointer to the VM.
2598	* @param pVCpu Pointer to the VMCPU.
2599	* @param pCtx Pointer to the guest-CPU context.
2600	*
2601	* @remarks No-long-jump zone!!!
2602	*/
2603	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2604	{
2605	int rc = VINF_SUCCESS;
2606	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2607	{
2608	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2609	Let us assert it as such and use native-width VMWRITE. */
2610	X86RFLAGS uRFlags = pCtx->rflags;
2611	Assert(uRFlags.u64 >> 32 == 0);
2612	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2613	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2614
2615	/*
2616	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2617	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2618	*/
2619	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2620	{
2621	Assert(pVM->hm.s.vmx.pRealModeTSS);
2622	Assert(PDMVmmDevHeapIsEnabled(pVM));
2623	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2624	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2625	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2626	}
2627
2628	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2629	AssertRCReturn(rc, rc);
2630
2631	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2632	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2633	}
2634	return rc;
2635	}
2636
2637
2638	/**
2639	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2640	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2641	* assembly code.
2642	*
2643	* @returns VBox status code.
2644	* @param pVM Pointer to the VM.
2645	* @param pVCpu Pointer to the VMCPU.
2646	* @param pCtx Pointer to the guest-CPU context.
2647	*
2648	* @remarks No-long-jump zone!!!
2649	*/
2650	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2651	{
2652	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2653	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2654	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2655	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2656	return rc;
2657	}
2658
2659
2660	/**
2661	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2662	* in the VMCS.
2663	*
2664	* @returns VBox status code.
2665	* @param pVM Pointer to the VM.
2666	* @param pVCpu Pointer to the VMCPU.
2667	* @param pCtx Pointer to the guest-CPU context.
2668	*
2669	* @remarks No-long-jump zone!!!
2670	*/
2671	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2672	{
2673	int rc = VINF_SUCCESS;
2674
2675	/*
2676	* Guest CR0.
2677	* Guest FPU.
2678	*/
2679	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2680	{
2681	uint64_t u64GuestCR0 = pCtx->cr0;
2682
2683	/* The guest's view (read access) of its CR0 is unblemished. */
2684	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2685	AssertRCReturn(rc, rc);
2686	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2687
2688	/* Setup VT-x's view of the guest CR0. */
2689	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2690	if (pVM->hm.s.fNestedPaging)
2691	{
2692	if (CPUMIsGuestPagingEnabledEx(pCtx))
2693	{
2694	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2695	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2696	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2697	}
2698	else
2699	{
2700	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2701	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2702	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2703	}
2704
2705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2706	AssertRCReturn(rc, rc);
2707	}
2708	else
2709	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2710
2711	/*
2712	* Guest FPU bits.
2713	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2714	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2715	*/
2716	u64GuestCR0 \|= X86_CR0_NE;
2717	bool fInterceptNM = false;
2718	if (CPUMIsGuestFPUStateActive(pVCpu))
2719	{
2720	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2721	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2722	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2723	}
2724	else
2725	{
2726	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2727	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2728	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2729	}
2730
2731	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2732	bool fInterceptMF = false;
2733	if (!(pCtx->cr0 & X86_CR0_NE))
2734	fInterceptMF = true;
2735
2736	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2737	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2738	{
2739	Assert(PDMVmmDevHeapIsEnabled(pVM));
2740	Assert(pVM->hm.s.vmx.pRealModeTSS);
2741	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2742	fInterceptNM = true;
2743	fInterceptMF = true;
2744	}
2745	else
2746	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2747
2748	if (fInterceptNM)
2749	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2750	else
2751	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2752
2753	if (fInterceptMF)
2754	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2755	else
2756	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2757
2758	/* Additional intercepts for debugging, define these yourself explicitly. */
2759	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2760	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2761	\| RT_BIT(X86_XCPT_DB)
2762	\| RT_BIT(X86_XCPT_DE)
2763	\| RT_BIT(X86_XCPT_NM)
2764	\| RT_BIT(X86_XCPT_UD)
2765	\| RT_BIT(X86_XCPT_NP)
2766	\| RT_BIT(X86_XCPT_SS)
2767	\| RT_BIT(X86_XCPT_GP)
2768	\| RT_BIT(X86_XCPT_PF)
2769	\| RT_BIT(X86_XCPT_MF);
2770	#elif defined(VBOX_ALWAYS_TRAP_PF)
2771	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2772	#endif
2773
2774	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2775
2776	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2777	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2778	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2779	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2780	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2781	else
2782	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2783
2784	u64GuestCR0 \|= uSetCR0;
2785	u64GuestCR0 &= uZapCR0;
2786	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2787
2788	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2789	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2790	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2791	Log(("Load: VMX_VMCS_GUEST_CR0=%RGv (uSetCR0=%#RGv uZapCR0=%#RGv)\n", u64GuestCR0, uSetCR0, uZapCR0));
2792
2793	/*
2794	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2795	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2796	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2797	*/
2798	uint64_t u64CR0Mask = 0;
2799	u64CR0Mask = X86_CR0_PE
2800	\| X86_CR0_NE
2801	\| X86_CR0_WP
2802	\| X86_CR0_PG
2803	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2804	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2805	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2806
2807	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2808	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2809	u64CR0Mask &= ~X86_CR0_PE;
2810
2811	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2812	if (fInterceptNM)
2813	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2814	else
2815	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2816
2817	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2818	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2819	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2820	AssertRCReturn(rc, rc);
2821
2822	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2823	}
2824
2825	/*
2826	* Guest CR2.
2827	* It's always loaded in the assembler code. Nothing to do here.
2828	*/
2829
2830	/*
2831	* Guest CR3.
2832	*/
2833	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2834	{
2835	uint64_t u64GuestCR3 = 0;
2836	if (pVM->hm.s.fNestedPaging)
2837	{
2838	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2839
2840	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2841	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2842	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2843	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2844
2845	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2846	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2847	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2848
2849	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2850	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2851	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2852	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2853
2854	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2855	AssertRCReturn(rc, rc);
2856	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2857
2858	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2859	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2860	{
2861	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2862	if (CPUMIsGuestInPAEModeEx(pCtx))
2863	{
2864	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2865	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2866	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2867	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2868	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2869	AssertRCReturn(rc, rc);
2870	}
2871
2872	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2873	have Unrestricted Execution to handle the guest when it's not using paging. */
2874	u64GuestCR3 = pCtx->cr3;
2875	}
2876	else
2877	{
2878	/*
2879	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2880	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2881	* EPT takes care of translating it to host-physical addresses.
2882	*/
2883	RTGCPHYS GCPhys;
2884	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2885	Assert(PDMVmmDevHeapIsEnabled(pVM));
2886
2887	/* We obtain it here every time as the guest could have relocated this PCI region. */
2888	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2889	AssertRCReturn(rc, rc);
2890
2891	u64GuestCR3 = GCPhys;
2892	}
2893	}
2894	else
2895	{
2896	/* Non-nested paging case, just use the hypervisor's CR3. */
2897	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2898	}
2899
2900	Log(("Load: VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2901	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2902	AssertRCReturn(rc, rc);
2903
2904	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2905	}
2906
2907	/*
2908	* Guest CR4.
2909	*/
2910	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2911	{
2912	uint64_t u64GuestCR4 = pCtx->cr4;
2913
2914	/* The guest's view of its CR4 is unblemished. */
2915	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2916	AssertRCReturn(rc, rc);
2917	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2918
2919	/* Setup VT-x's view of the guest CR4. */
2920	/*
2921	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2922	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2923	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2924	*/
2925	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2926	{
2927	Assert(pVM->hm.s.vmx.pRealModeTSS);
2928	Assert(PDMVmmDevHeapIsEnabled(pVM));
2929	u64GuestCR4 &= ~X86_CR4_VME;
2930	}
2931
2932	if (pVM->hm.s.fNestedPaging)
2933	{
2934	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2935	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2936	{
2937	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2938	u64GuestCR4 \|= X86_CR4_PSE;
2939	/* Our identity mapping is a 32 bits page directory. */
2940	u64GuestCR4 &= ~X86_CR4_PAE;
2941	}
2942	/* else use guest CR4.*/
2943	}
2944	else
2945	{
2946	/*
2947	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2948	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2949	*/
2950	switch (pVCpu->hm.s.enmShadowMode)
2951	{
2952	case PGMMODE_REAL: /* Real-mode. */
2953	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2954	case PGMMODE_32_BIT: /* 32-bit paging. */
2955	{
2956	u64GuestCR4 &= ~X86_CR4_PAE;
2957	break;
2958	}
2959
2960	case PGMMODE_PAE: /* PAE paging. */
2961	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2962	{
2963	u64GuestCR4 \|= X86_CR4_PAE;
2964	break;
2965	}
2966
2967	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2968	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2969	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2970	break;
2971	#endif
2972	default:
2973	AssertFailed();
2974	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2975	}
2976	}
2977
2978	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2979	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2980	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2981	u64GuestCR4 \|= uSetCR4;
2982	u64GuestCR4 &= uZapCR4;
2983
2984	/* Write VT-x's view of the guest CR4 into the VMCS. */
2985	Log(("Load: VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2986	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2987
2988	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2989	uint64_t u64CR4Mask = 0;
2990	u64CR4Mask = X86_CR4_VME
2991	\| X86_CR4_PAE
2992	\| X86_CR4_PGE
2993	\| X86_CR4_PSE
2994	\| X86_CR4_VMXE;
2995	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2996	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2997	AssertRCReturn(rc, rc);
2998
2999	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3000	}
3001	return rc;
3002	}
3003
3004
3005	/**
3006	* Loads the guest debug registers into the guest-state area in the VMCS.
3007	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3008	*
3009	* @returns VBox status code.
3010	* @param pVM Pointer to the VM.
3011	* @param pVCpu Pointer to the VMCPU.
3012	* @param pCtx Pointer to the guest-CPU context.
3013	*
3014	* @remarks No-long-jump zone!!!
3015	*/
3016	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3017	{
3018	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3019	return VINF_SUCCESS;
3020
3021	#ifdef VBOX_STRICT
3022	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3023	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3024	{
3025	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3026	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3027	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3028	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3029	}
3030	#endif
3031
3032	int rc = VERR_INTERNAL_ERROR_5;
3033	bool fInterceptDB = false;
3034	bool fInterceptMovDRx = false;
3035	if (DBGFIsStepping(pVCpu))
3036	{
3037	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
3038	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3039	{
3040	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3041	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3042	AssertRCReturn(rc, rc);
3043	Assert(fInterceptDB == false);
3044	}
3045	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3046	{
3047	Assert(fInterceptDB == false);
3048	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3049	* just continuing? */
3050	}
3051	else
3052	fInterceptDB = true;
3053	}
3054	else
3055	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3056
3057	/*
3058	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3059	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3060	* The same for the hypervisor DRx registers, priority is for the guest here.
3061	*/
3062	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3063	&& !CPUMIsGuestDebugStateActive(pVCpu))
3064	{
3065	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3066	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3067	AssertRC(rc);
3068	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3069	Assert(fInterceptMovDRx == false);
3070	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3071	}
3072	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3073	&& !CPUMIsHyperDebugStateActive(pVCpu))
3074	{
3075	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3076	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3077	AssertRC(rc);
3078	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3079	fInterceptMovDRx = true;
3080	}
3081	else
3082	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3083
3084	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3085	if (fInterceptDB)
3086	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3087	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3088	{
3089	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3090	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3091	#endif
3092	}
3093
3094	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3095	if (fInterceptMovDRx)
3096	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3097	else
3098	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3099
3100	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3101	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3102
3103	/* The guest's view of its DR7 is unblemished. */
3104	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3105
3106	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3107	return rc;
3108	}
3109
3110
3111	#ifdef VBOX_STRICT
3112	/**
3113	* Strict function to validate segment registers.
3114	*/
3115	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3116	{
3117	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3118	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3119	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3120	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3121	{
3122	/* Protected mode checks */
3123	/* CS */
3124	Assert(pCtx->cs.Attr.n.u1Present);
3125	Assert(!(pCtx->cs.Attr.u & 0xf00));
3126	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3127	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3128	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3129	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3130	\|\| (pCtx->cs.Attr.n.u1Granularity));
3131	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3132	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3133	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3134	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3135	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3136	else
3137	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3138	/* SS */
3139	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3140	{
3141	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3142	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3143	Assert(pCtx->ss.Attr.n.u1Present);
3144	Assert(!(pCtx->ss.Attr.u & 0xf00));
3145	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3146	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3147	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3148	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3149	\|\| (pCtx->ss.Attr.n.u1Granularity));
3150	}
3151	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3152	/* CR0 might not be up-to-date here always, hence disabled. */
3153	#if 0
3154	if (!pCtx->cr0 & X86_CR0_PE)
3155	Assert(!pCtx->ss.Attr.n.u2Dpl);
3156	#endif
3157	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3158	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3159	{
3160	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3161	Assert(pCtx->ds.Attr.n.u1Present);
3162	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3163	Assert(!(pCtx->ds.Attr.u & 0xf00));
3164	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3165	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3166	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3167	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3168	\|\| (pCtx->ds.Attr.n.u1Granularity));
3169	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3170	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3171	}
3172	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3173	{
3174	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3175	Assert(pCtx->es.Attr.n.u1Present);
3176	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3177	Assert(!(pCtx->es.Attr.u & 0xf00));
3178	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3179	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3180	\|\| !(pCtx->es.Attr.n.u1Granularity));
3181	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3182	\|\| (pCtx->es.Attr.n.u1Granularity));
3183	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3184	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3185	}
3186	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3187	{
3188	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3189	Assert(pCtx->fs.Attr.n.u1Present);
3190	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3191	Assert(!(pCtx->fs.Attr.u & 0xf00));
3192	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3193	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3194	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3195	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3196	\|\| (pCtx->fs.Attr.n.u1Granularity));
3197	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3198	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3199	}
3200	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3201	{
3202	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3203	Assert(pCtx->gs.Attr.n.u1Present);
3204	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3205	Assert(!(pCtx->gs.Attr.u & 0xf00));
3206	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3207	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3208	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3209	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3210	\|\| (pCtx->gs.Attr.n.u1Granularity));
3211	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3212	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3213	}
3214	/* 64-bit capable CPUs. */
3215	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3216	Assert(!(pCtx->cs.u64Base >> 32));
3217	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3218	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3219	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3220	# endif
3221	}
3222	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3223	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3224	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3225	{
3226	/* Real and v86 mode checks. */
3227	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3228	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3229	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3230	{
3231	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3232	}
3233	else
3234	{
3235	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3236	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3237	}
3238
3239	/* CS */
3240	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3241	Assert(pCtx->cs.u32Limit == 0xffff);
3242	Assert(u32CSAttr == 0xf3);
3243	/* SS */
3244	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3245	Assert(pCtx->ss.u32Limit == 0xffff);
3246	Assert(u32SSAttr == 0xf3);
3247	/* DS */
3248	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3249	Assert(pCtx->ds.u32Limit == 0xffff);
3250	Assert(u32DSAttr == 0xf3);
3251	/* ES */
3252	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3253	Assert(pCtx->es.u32Limit == 0xffff);
3254	Assert(u32ESAttr == 0xf3);
3255	/* FS */
3256	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3257	Assert(pCtx->fs.u32Limit == 0xffff);
3258	Assert(u32FSAttr == 0xf3);
3259	/* GS */
3260	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3261	Assert(pCtx->gs.u32Limit == 0xffff);
3262	Assert(u32GSAttr == 0xf3);
3263	/* 64-bit capable CPUs. */
3264	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3265	Assert(!(pCtx->cs.u64Base >> 32));
3266	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3267	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3268	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3269	# endif
3270	}
3271	}
3272	#endif /* VBOX_STRICT */
3273
3274
3275	/**
3276	* Writes a guest segment register into the guest-state area in the VMCS.
3277	*
3278	* @returns VBox status code.
3279	* @param pVM Pointer to the VM.
3280	* @param pVCpu Pointer to the VMCPU.
3281	* @param idxSel Index of the selector in the VMCS.
3282	* @param idxLimit Index of the segment limit in the VMCS.
3283	* @param idxBase Index of the segment base in the VMCS.
3284	* @param idxAccess Index of the access rights of the segment in the VMCS.
3285	* @param pSelReg Pointer to the segment selector.
3286	* @param pCtx Pointer to the guest-CPU context.
3287	*
3288	* @remarks No-long-jump zone!!!
3289	*/
3290	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3291	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3292	{
3293	int rc;
3294	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3295	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3296	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3297	AssertRCReturn(rc, rc);
3298
3299	uint32_t u32Access = pSelReg->Attr.u;
3300	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3301	{
3302	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3303	u32Access = 0xf3;
3304	Assert(pVM->hm.s.vmx.pRealModeTSS);
3305	Assert(PDMVmmDevHeapIsEnabled(pVM));
3306	}
3307	else
3308	{
3309	/*
3310	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3311	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3312	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3313	* loaded in protected-mode have their attribute as 0.
3314	*/
3315	if (!u32Access)
3316	u32Access = VMX_SEL_UNUSABLE;
3317	}
3318
3319	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3320	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3321	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3322
3323	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3324	AssertRCReturn(rc, rc);
3325	return rc;
3326	}
3327
3328
3329	/**
3330	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3331	* into the guest-state area in the VMCS.
3332	*
3333	* @returns VBox status code.
3334	* @param pVM Pointer to the VM.
3335	* @param pVCPU Pointer to the VMCPU.
3336	* @param pCtx Pointer to the guest-CPU context.
3337	*
3338	* @remarks No-long-jump zone!!!
3339	*/
3340	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3341	{
3342	int rc = VERR_INTERNAL_ERROR_5;
3343
3344	/*
3345	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3346	*/
3347	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3348	{
3349	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3350	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3351	{
3352	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3353	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3354	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3355	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3356	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3357	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3358	}
3359
3360	#ifdef VBOX_WITH_REM
3361	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3362	{
3363	Assert(pVM->hm.s.vmx.pRealModeTSS);
3364	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3365	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3366	{
3367	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3368	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3369	&& enmGuestMode >= PGMMODE_PROTECTED)
3370	{
3371	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3372	in real-mode (e.g. OpenBSD 4.0) */
3373	REMFlushTBs(pVM);
3374	Log(("Load: Switch to protected mode detected!\n"));
3375	}
3376	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3377	}
3378	}
3379	#endif
3380	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3381	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3382	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3383	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3384	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3385	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3386	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3387	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3388	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3389	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3390	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3391	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3392	AssertRCReturn(rc, rc);
3393
3394	#ifdef VBOX_STRICT
3395	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3396	#endif
3397	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3398	}
3399
3400	/*
3401	* Guest TR.
3402	*/
3403	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3404	{
3405	/*
3406	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3407	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3408	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3409	*/
3410	uint16_t u16Sel = 0;
3411	uint32_t u32Limit = 0;
3412	uint64_t u64Base = 0;
3413	uint32_t u32AccessRights = 0;
3414
3415	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3416	{
3417	u16Sel = pCtx->tr.Sel;
3418	u32Limit = pCtx->tr.u32Limit;
3419	u64Base = pCtx->tr.u64Base;
3420	u32AccessRights = pCtx->tr.Attr.u;
3421	}
3422	else
3423	{
3424	Assert(pVM->hm.s.vmx.pRealModeTSS);
3425	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3426
3427	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3428	RTGCPHYS GCPhys;
3429	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3430	AssertRCReturn(rc, rc);
3431
3432	X86DESCATTR DescAttr;
3433	DescAttr.u = 0;
3434	DescAttr.n.u1Present = 1;
3435	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3436
3437	u16Sel = 0;
3438	u32Limit = HM_VTX_TSS_SIZE;
3439	u64Base = GCPhys; /* in real-mode phys = virt. */
3440	u32AccessRights = DescAttr.u;
3441	}
3442
3443	/* Validate. */
3444	Assert(!(u16Sel & RT_BIT(2)));
3445	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3446	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3447	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3448	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3449	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3450	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3451	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3452	Assert( (u32Limit & 0xfff) == 0xfff
3453	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3454	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3455	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3456
3457	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3458	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3459	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3460	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3461	AssertRCReturn(rc, rc);
3462
3463	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3464	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3465	}
3466
3467	/*
3468	* Guest GDTR.
3469	*/
3470	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3471	{
3472	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3473	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3474	AssertRCReturn(rc, rc);
3475
3476	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3477	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3478	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3479	}
3480
3481	/*
3482	* Guest LDTR.
3483	*/
3484	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3485	{
3486	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3487	uint32_t u32Access = 0;
3488	if (!pCtx->ldtr.Attr.u)
3489	u32Access = VMX_SEL_UNUSABLE;
3490	else
3491	u32Access = pCtx->ldtr.Attr.u;
3492
3493	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3494	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3495	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3496	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3497	AssertRCReturn(rc, rc);
3498
3499	/* Validate. */
3500	if (!(u32Access & VMX_SEL_UNUSABLE))
3501	{
3502	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3503	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3504	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3505	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3506	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3507	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3508	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3509	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3510	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3511	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3512	}
3513
3514	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3515	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3516	}
3517
3518	/*
3519	* Guest IDTR.
3520	*/
3521	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3522	{
3523	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3524	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3525	AssertRCReturn(rc, rc);
3526
3527	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3528	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3529	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3530	}
3531
3532	/*
3533	* Guest FS & GS base MSRs.
3534	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3535	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3536	* are only available in 64-bit mode.
3537	*/
3538	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3539	* should not be necessary to do it in assembly again. */
3540	if (CPUMIsGuestInLongModeEx(pCtx))
3541	{
3542	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3543	{
3544	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3545	AssertRCReturn(rc, rc);
3546	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3547	}
3548
3549	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3550	{
3551	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3552	AssertRCReturn(rc, rc);
3553	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3554	}
3555	}
3556	else
3557	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3558
3559	return VINF_SUCCESS;
3560	}
3561
3562
3563	/**
3564	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3565	* areas. These MSRs will automatically be loaded to the host CPU on every
3566	* successful VM entry and stored from the host CPU on every successful VM exit.
3567	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3568	*
3569	* @returns VBox status code.
3570	* @param pVM Pointer to the VM.
3571	* @param pVCpu Pointer to the VMCPU.
3572	* @param pCtx Pointer to the guest-CPU context.
3573	*
3574	* @remarks No-long-jump zone!!!
3575	*/
3576	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3577	{
3578	AssertPtr(pVCpu);
3579	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3580
3581	/*
3582	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3583	*/
3584	int rc = VINF_SUCCESS;
3585	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3586	{
3587	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3588	uint32_t cGuestMsrs = 0;
3589
3590	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3591	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3592	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3593	if (fSupportsNX \|\| fSupportsLongMode)
3594	{
3595	/** @todo support save IA32_EFER, i.e.
3596	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3597	* guest EFER need not be part of the VM-entry MSR-load area. */
3598	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3599	pGuestMsr->u32Reserved = 0;
3600	pGuestMsr->u64Value = pCtx->msrEFER;
3601	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3602	if (!CPUMIsGuestInLongModeEx(pCtx))
3603	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3604	pGuestMsr++; cGuestMsrs++;
3605	if (fSupportsLongMode)
3606	{
3607	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3608	pGuestMsr->u32Reserved = 0;
3609	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3610	pGuestMsr++; cGuestMsrs++;
3611	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3612	pGuestMsr->u32Reserved = 0;
3613	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3614	pGuestMsr++; cGuestMsrs++;
3615	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3618	pGuestMsr++; cGuestMsrs++;
3619	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3620	#if 0
3621	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3622	pGuestMsr->u32Reserved = 0;
3623	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3624	pGuestMsr++; cGuestMsrs++;
3625	#endif
3626	}
3627	}
3628
3629	/*
3630	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3631	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3632	*/
3633	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3634	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3635	{
3636	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3637	pGuestMsr->u32Reserved = 0;
3638	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3639	AssertRCReturn(rc, rc);
3640	pGuestMsr++; cGuestMsrs++;
3641	}
3642
3643	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3644	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3645	{
3646	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3647	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3648	}
3649
3650	/* Update the VCPU's copy of the guest MSR count. */
3651	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3652	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3653	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3654	AssertRCReturn(rc, rc);
3655
3656	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3657	}
3658
3659	/*
3660	* Guest Sysenter MSRs.
3661	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3662	* VM exits on WRMSRs for these MSRs.
3663	*/
3664	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3665	{
3666	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3667	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3668	}
3669	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3670	{
3671	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3672	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3673	}
3674	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3675	{
3676	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3677	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3678	}
3679	AssertRCReturn(rc, rc);
3680
3681	return rc;
3682	}
3683
3684
3685	/**
3686	* Loads the guest activity state into the guest-state area in the VMCS.
3687	*
3688	* @returns VBox status code.
3689	* @param pVM Pointer to the VM.
3690	* @param pVCpu Pointer to the VMCPU.
3691	* @param pCtx Pointer to the guest-CPU context.
3692	*
3693	* @remarks No-long-jump zone!!!
3694	*/
3695	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3696	{
3697	/** @todo See if we can make use of other states, e.g.
3698	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3699	int rc = VINF_SUCCESS;
3700	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3701	{
3702	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3703	AssertRCReturn(rc, rc);
3704	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3705	}
3706	return rc;
3707	}
3708
3709
3710	/**
3711	* Sets up the appropriate function to run guest code.
3712	*
3713	* @returns VBox status code.
3714	* @param pVM Pointer to the VM.
3715	* @param pVCpu Pointer to the VMCPU.
3716	* @param pCtx Pointer to the guest-CPU context.
3717	*
3718	* @remarks No-long-jump zone!!!
3719	*/
3720	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3721	{
3722	if (CPUMIsGuestInLongModeEx(pCtx))
3723	{
3724	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3725	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3726	#endif
3727	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3728	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3729	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3730	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3731	#else
3732	/* 64-bit host or hybrid host. */
3733	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3734	#endif
3735	}
3736	else
3737	{
3738	/* Guest is not in long mode, use the 32-bit handler. */
3739	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3740	}
3741	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3742	return VINF_SUCCESS;
3743	}
3744
3745
3746	/**
3747	* Wrapper for running the guest code in VT-x.
3748	*
3749	* @returns VBox strict status code.
3750	* @param pVM Pointer to the VM.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param pCtx Pointer to the guest-CPU context.
3753	*
3754	* @remarks No-long-jump zone!!!
3755	*/
3756	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3757	{
3758	/*
3759	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3760	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3761	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3762	*/
3763	#ifdef VBOX_WITH_KERNEL_USING_XMM
3764	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3765	#else
3766	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3767	#endif
3768	}
3769
3770
3771	/**
3772	* Report world-switch error and dump some useful debug info.
3773	*
3774	* @param pVM Pointer to the VM.
3775	* @param pVCpu Pointer to the VMCPU.
3776	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3777	* @param pCtx Pointer to the guest-CPU context.
3778	* @param pVmxTransient Pointer to the VMX transient structure (only
3779	* exitReason updated).
3780	*/
3781	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3782	{
3783	Assert(pVM);
3784	Assert(pVCpu);
3785	Assert(pCtx);
3786	Assert(pVmxTransient);
3787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3788
3789	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3790	switch (rcVMRun)
3791	{
3792	case VERR_VMX_INVALID_VMXON_PTR:
3793	AssertFailed();
3794	break;
3795	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3796	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3797	{
3798	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3799	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3800	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3801	AssertRC(rc);
3802
3803	#ifdef VBOX_STRICT
3804	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3805	pVmxTransient->uExitReason));
3806	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3807	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3808	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3809	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3810	else
3811	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3812
3813	/* VMX control bits. */
3814	uint32_t u32Val;
3815	uint64_t u64Val;
3816	RTHCUINTREG uHCReg;
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3845	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3846	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3847	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3848	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3849	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3850	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3852	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3853	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3854	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3855	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3856	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3857	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3858	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3859
3860	/* Guest bits. */
3861	RTGCUINTREG uGCReg;
3862	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3863	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3864	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3865	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3866	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3867	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3868	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3869	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3870
3871	/* Host bits. */
3872	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3873	Log(("Host CR0 %#RHr\n", uHCReg));
3874	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3875	Log(("Host CR3 %#RHr\n", uHCReg));
3876	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3877	Log(("Host CR4 %#RHr\n", uHCReg));
3878
3879	RTGDTR HostGdtr;
3880	PCX86DESCHC pDesc;
3881	ASMGetGDTR(&HostGdtr);
3882	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3883	Log(("Host CS %#08x\n", u32Val));
3884	if (u32Val < HostGdtr.cbGdt)
3885	{
3886	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3887	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3888	}
3889
3890	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3891	Log(("Host DS %#08x\n", u32Val));
3892	if (u32Val < HostGdtr.cbGdt)
3893	{
3894	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3895	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3896	}
3897
3898	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3899	Log(("Host ES %#08x\n", u32Val));
3900	if (u32Val < HostGdtr.cbGdt)
3901	{
3902	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3903	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3904	}
3905
3906	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3907	Log(("Host FS %#08x\n", u32Val));
3908	if (u32Val < HostGdtr.cbGdt)
3909	{
3910	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3911	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3912	}
3913
3914	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3915	Log(("Host GS %#08x\n", u32Val));
3916	if (u32Val < HostGdtr.cbGdt)
3917	{
3918	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3919	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3920	}
3921
3922	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3923	Log(("Host SS %#08x\n", u32Val));
3924	if (u32Val < HostGdtr.cbGdt)
3925	{
3926	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3927	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3928	}
3929
3930	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3931	Log(("Host TR %#08x\n", u32Val));
3932	if (u32Val < HostGdtr.cbGdt)
3933	{
3934	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3935	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3936	}
3937
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3939	Log(("Host TR Base %#RHv\n", uHCReg));
3940	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3941	Log(("Host GDTR Base %#RHv\n", uHCReg));
3942	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3943	Log(("Host IDTR Base %#RHv\n", uHCReg));
3944	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3945	Log(("Host SYSENTER CS %#08x\n", u32Val));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3947	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3949	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3950	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3951	Log(("Host RSP %#RHv\n", uHCReg));
3952	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3953	Log(("Host RIP %#RHv\n", uHCReg));
3954	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3955	if (VMX_IS_64BIT_HOST_MODE())
3956	{
3957	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3958	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3959	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3960	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3961	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3962	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3963	}
3964	# endif
3965	#endif /* VBOX_STRICT */
3966	break;
3967	}
3968
3969	default:
3970	/* Impossible */
3971	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3972	break;
3973	}
3974	NOREF(pVM);
3975	}
3976
3977
3978	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3979	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3980	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3981	#endif
3982
3983	#ifdef VBOX_STRICT
3984	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3985	{
3986	switch (idxField)
3987	{
3988	case VMX_VMCS_GUEST_RIP:
3989	case VMX_VMCS_GUEST_RSP:
3990	case VMX_VMCS_GUEST_RFLAGS:
3991	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3992	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3993	case VMX_VMCS_GUEST_CR0:
3994	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3995	case VMX_VMCS_GUEST_CR4:
3996	case VMX_VMCS_GUEST_DR7:
3997	case VMX_VMCS32_GUEST_SYSENTER_CS:
3998	case VMX_VMCS_GUEST_SYSENTER_EIP:
3999	case VMX_VMCS_GUEST_SYSENTER_ESP:
4000	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4001	case VMX_VMCS_GUEST_GDTR_BASE:
4002	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4003	case VMX_VMCS_GUEST_IDTR_BASE:
4004	case VMX_VMCS16_GUEST_FIELD_CS:
4005	case VMX_VMCS32_GUEST_CS_LIMIT:
4006	case VMX_VMCS_GUEST_CS_BASE:
4007	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4008	case VMX_VMCS16_GUEST_FIELD_DS:
4009	case VMX_VMCS32_GUEST_DS_LIMIT:
4010	case VMX_VMCS_GUEST_DS_BASE:
4011	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4012	case VMX_VMCS16_GUEST_FIELD_ES:
4013	case VMX_VMCS32_GUEST_ES_LIMIT:
4014	case VMX_VMCS_GUEST_ES_BASE:
4015	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4016	case VMX_VMCS16_GUEST_FIELD_FS:
4017	case VMX_VMCS32_GUEST_FS_LIMIT:
4018	case VMX_VMCS_GUEST_FS_BASE:
4019	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4020	case VMX_VMCS16_GUEST_FIELD_GS:
4021	case VMX_VMCS32_GUEST_GS_LIMIT:
4022	case VMX_VMCS_GUEST_GS_BASE:
4023	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4024	case VMX_VMCS16_GUEST_FIELD_SS:
4025	case VMX_VMCS32_GUEST_SS_LIMIT:
4026	case VMX_VMCS_GUEST_SS_BASE:
4027	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4028	case VMX_VMCS16_GUEST_FIELD_LDTR:
4029	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4030	case VMX_VMCS_GUEST_LDTR_BASE:
4031	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4032	case VMX_VMCS16_GUEST_FIELD_TR:
4033	case VMX_VMCS32_GUEST_TR_LIMIT:
4034	case VMX_VMCS_GUEST_TR_BASE:
4035	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4036	case VMX_VMCS32_RO_EXIT_REASON:
4037	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4038	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4039	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4040	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4041	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4042	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4043	case VMX_VMCS32_RO_IDT_INFO:
4044	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4045	case VMX_VMCS_GUEST_CR3:
4046	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4047	return true;
4048	}
4049	return false;
4050	}
4051
4052	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4053	{
4054	switch (idxField)
4055	{
4056	case VMX_VMCS_GUEST_LDTR_BASE:
4057	case VMX_VMCS_GUEST_TR_BASE:
4058	case VMX_VMCS_GUEST_GDTR_BASE:
4059	case VMX_VMCS_GUEST_IDTR_BASE:
4060	case VMX_VMCS_GUEST_SYSENTER_EIP:
4061	case VMX_VMCS_GUEST_SYSENTER_ESP:
4062	case VMX_VMCS_GUEST_CR0:
4063	case VMX_VMCS_GUEST_CR4:
4064	case VMX_VMCS_GUEST_CR3:
4065	case VMX_VMCS_GUEST_DR7:
4066	case VMX_VMCS_GUEST_RIP:
4067	case VMX_VMCS_GUEST_RSP:
4068	case VMX_VMCS_GUEST_CS_BASE:
4069	case VMX_VMCS_GUEST_DS_BASE:
4070	case VMX_VMCS_GUEST_ES_BASE:
4071	case VMX_VMCS_GUEST_FS_BASE:
4072	case VMX_VMCS_GUEST_GS_BASE:
4073	case VMX_VMCS_GUEST_SS_BASE:
4074	return true;
4075	}
4076	return false;
4077	}
4078	#endif /* VBOX_STRICT */
4079
4080	/**
4081	* Executes the specified handler in 64-bit mode.
4082	*
4083	* @returns VBox status code.
4084	* @param pVM Pointer to the VM.
4085	* @param pVCpu Pointer to the VMCPU.
4086	* @param pCtx Pointer to the guest CPU context.
4087	* @param pfnHandler Pointer to the RC handler function.
4088	* @param cbParam Number of parameters.
4089	* @param paParam Array of 32-bit parameters.
4090	*/
4091	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4092	uint32_t *paParam)
4093	{
4094	int rc, rc2;
4095	PHMGLOBLCPUINFO pCpu;
4096	RTHCPHYS HCPhysCpuPage;
4097	RTHCUINTREG uOldEFlags;
4098
4099	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4100	Assert(pfnHandler);
4101	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4102	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4103
4104	#ifdef VBOX_STRICT
4105	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4106	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4107
4108	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4109	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4110	#endif
4111
4112	/* Disable interrupts. */
4113	uOldEFlags = ASMIntDisableFlags();
4114
4115	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4116	RTCPUID idHostCpu = RTMpCpuId();
4117	CPUMR0SetLApic(pVM, idHostCpu);
4118	#endif
4119
4120	pCpu = HMR0GetCurrentCpu();
4121	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4122
4123	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4124	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4125
4126	/* Leave VMX Root Mode. */
4127	VMXDisable();
4128
4129	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4130
4131	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4132	CPUMSetHyperEIP(pVCpu, pfnHandler);
4133	for (int i = (int)cbParam - 1; i >= 0; i--)
4134	CPUMPushHyper(pVCpu, paParam[i]);
4135
4136	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4137
4138	/* Call the switcher. */
4139	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4140	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4141
4142	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4143	/* Make sure the VMX instructions don't cause #UD faults. */
4144	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4145
4146	/* Re-enter VMX Root Mode */
4147	rc2 = VMXEnable(HCPhysCpuPage);
4148	if (RT_FAILURE(rc2))
4149	{
4150	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4151	ASMSetFlags(uOldEFlags);
4152	return rc2;
4153	}
4154
4155	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4156	AssertRC(rc2);
4157	Assert(!(ASMGetFlags() & X86_EFL_IF));
4158	ASMSetFlags(uOldEFlags);
4159	return rc;
4160	}
4161
4162
4163	/**
4164	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4165	* supporting 64-bit guests.
4166	*
4167	* @returns VBox status code.
4168	* @param fResume Whether to VMLAUNCH or VMRESUME.
4169	* @param pCtx Pointer to the guest-CPU context.
4170	* @param pCache Pointer to the VMCS cache.
4171	* @param pVM Pointer to the VM.
4172	* @param pVCpu Pointer to the VMCPU.
4173	*/
4174	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4175	{
4176	uint32_t aParam[6];
4177	PHMGLOBLCPUINFO pCpu = NULL;
4178	RTHCPHYS HCPhysCpuPage = 0;
4179	int rc = VERR_INTERNAL_ERROR_5;
4180
4181	pCpu = HMR0GetCurrentCpu();
4182	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4183
4184	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4185	pCache->uPos = 1;
4186	pCache->interPD = PGMGetInterPaeCR3(pVM);
4187	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4188	#endif
4189
4190	#ifdef VBOX_STRICT
4191	pCache->TestIn.HCPhysCpuPage = 0;
4192	pCache->TestIn.HCPhysVmcs = 0;
4193	pCache->TestIn.pCache = 0;
4194	pCache->TestOut.HCPhysVmcs = 0;
4195	pCache->TestOut.pCache = 0;
4196	pCache->TestOut.pCtx = 0;
4197	pCache->TestOut.eflags = 0;
4198	#endif
4199
4200	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4201	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4202	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4203	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4204	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4205	aParam[5] = 0;
4206
4207	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4208	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4209	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4210	#endif
4211	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4212
4213	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4214	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4215	Assert(pCtx->dr[4] == 10);
4216	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4217	#endif
4218
4219	#ifdef VBOX_STRICT
4220	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4221	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4222	pVCpu->hm.s.vmx.HCPhysVmcs));
4223	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4224	pCache->TestOut.HCPhysVmcs));
4225	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4226	pCache->TestOut.pCache));
4227	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4228	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4229	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4230	pCache->TestOut.pCtx));
4231	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4232	#endif
4233	return rc;
4234	}
4235
4236
4237	/**
4238	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4239	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4240	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4241	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4242	*
4243	* @returns VBox status code.
4244	* @param pVM Pointer to the VM.
4245	* @param pVCpu Pointer to the VMCPU.
4246	*/
4247	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4248	{
4249	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4250	{ \
4251	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4252	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4253	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4254	}
4255
4256	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4257	{ \
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4262	}
4263
4264	AssertPtr(pVM);
4265	AssertPtr(pVCpu);
4266	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4267
4268	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4269	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4270	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4271	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4272	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4273	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4274	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4275	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4276	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4277
4278	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4279	#if 0
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4289	#endif
4290
4291	/* 32-bit guest-state fields. */
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4296	/* Unused 32-bit guest-state fields. */
4297	#if 0
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4301	#endif
4302
4303	/* Natural width guest-state fields. */
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4320	/* Unused natural width guest-state fields. */
4321	#if 0
4322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4324	#endif
4325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4327
4328	if (pVM->hm.s.fNestedPaging)
4329	{
4330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4332	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4333	}
4334	else
4335	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4336
4337	#undef VMXLOCAL_INIT_VMCS_SELREG
4338	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4339	return VINF_SUCCESS;
4340	}
4341
4342
4343	/**
4344	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4345	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4346	* darwin, running 64-bit guests).
4347	*
4348	* @returns VBox status code.
4349	* @param pVCpu Pointer to the VMCPU.
4350	* @param idxField The VMCS field encoding.
4351	* @param u64Val 16, 32 or 64 bits value.
4352	*/
4353	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4354	{
4355	int rc;
4356	switch (idxField)
4357	{
4358	/*
4359	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4360	*/
4361	/* 64-bit Control fields. */
4362	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4363	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4364	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4365	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4366	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4367	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4368	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4369	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4370	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4371	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4372	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4373	case VMX_VMCS64_CTRL_EPTP_FULL:
4374	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4375	/* 64-bit Read-only data fields. */
4376	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4377	/* 64-bit Guest-state fields. */
4378	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4379	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4380	case VMX_VMCS64_GUEST_PAT_FULL:
4381	case VMX_VMCS64_GUEST_EFER_FULL:
4382	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4383	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4384	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4385	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4386	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4387	/* 64-bit Host-state fields. */
4388	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4389	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4390	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4391	{
4392	rc = VMXWriteVmcs32(idxField, u64Val);
4393	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4394	break;
4395	}
4396
4397	/*
4398	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4399	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4400	*/
4401	/* Natural-width Guest-state fields. */
4402	case VMX_VMCS_GUEST_CR0:
4403	case VMX_VMCS_GUEST_CR3:
4404	case VMX_VMCS_GUEST_CR4:
4405	case VMX_VMCS_GUEST_ES_BASE:
4406	case VMX_VMCS_GUEST_CS_BASE:
4407	case VMX_VMCS_GUEST_SS_BASE:
4408	case VMX_VMCS_GUEST_DS_BASE:
4409	case VMX_VMCS_GUEST_FS_BASE:
4410	case VMX_VMCS_GUEST_GS_BASE:
4411	case VMX_VMCS_GUEST_LDTR_BASE:
4412	case VMX_VMCS_GUEST_TR_BASE:
4413	case VMX_VMCS_GUEST_GDTR_BASE:
4414	case VMX_VMCS_GUEST_IDTR_BASE:
4415	case VMX_VMCS_GUEST_DR7:
4416	case VMX_VMCS_GUEST_RSP:
4417	case VMX_VMCS_GUEST_RIP:
4418	case VMX_VMCS_GUEST_RFLAGS:
4419	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4420	case VMX_VMCS_GUEST_SYSENTER_ESP:
4421	case VMX_VMCS_GUEST_SYSENTER_EIP:
4422	{
4423	if ((u64Val >> 32ULL) == 0)
4424	{
4425	/* If this field is 64-bit, VT-x will zero out the top bits. */
4426	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4427	}
4428	else
4429	{
4430	/* Assert that only the 32->64 switcher case should ever come here. */
4431	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4432	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4433	}
4434	break;
4435	}
4436
4437	default:
4438	{
4439	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4440	rc = VERR_INVALID_PARAMETER;
4441	break;
4442	}
4443	}
4444	AssertRCReturn(rc, rc);
4445	return rc;
4446	}
4447
4448
4449	/**
4450	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4451	* hosts (except darwin) for 64-bit guests.
4452	*
4453	* @param pVCpu Pointer to the VMCPU.
4454	* @param idxField The VMCS field encoding.
4455	* @param u64Val 16, 32 or 64 bits value.
4456	*/
4457	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4458	{
4459	AssertPtr(pVCpu);
4460	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4461
4462	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4463	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4464
4465	/* Make sure there are no duplicates. */
4466	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4467	{
4468	if (pCache->Write.aField[i] == idxField)
4469	{
4470	pCache->Write.aFieldVal[i] = u64Val;
4471	return VINF_SUCCESS;
4472	}
4473	}
4474
4475	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4476	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4477	pCache->Write.cValidEntries++;
4478	return VINF_SUCCESS;
4479	}
4480
4481
4482	/**
4483	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4484	*
4485	* @param pVCpu Pointer to the VMCPU.
4486	* @param pCache Pointer to the VMCS cache.
4487	*
4488	* @remarks No-long-jump zone!!!
4489	*/
4490	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4491	{
4492	AssertPtr(pCache);
4493	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4494	{
4495	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4496	AssertRC(rc);
4497	}
4498	pCache->Write.cValidEntries = 0;
4499	}
4500
4501
4502	/**
4503	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4504	*
4505	* @param pVCpu Pointer to the VMCPU.
4506	* @param pCache Pointer to the VMCS cache.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4511	{
4512	AssertPtr(pCache);
4513	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4514	{
4515	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4516	AssertRC(rc);
4517	}
4518	}
4519	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4520
4521
4522	/**
4523	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4524	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4525	* timer.
4526	*
4527	* @returns VBox status code.
4528	* @param pVM Pointer to the VM.
4529	* @param pVCpu Pointer to the VMCPU.
4530	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4531	* out-of-sync. Make sure to update the required fields
4532	* before using them.
4533	* @remarks No-long-jump zone!!!
4534	*/
4535	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4536	{
4537	int rc = VERR_INTERNAL_ERROR_5;
4538	bool fOffsettedTsc = false;
4539	if (pVM->hm.s.vmx.fUsePreemptTimer)
4540	{
4541	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4542
4543	/* Make sure the returned values have sane upper and lower boundaries. */
4544	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4545	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4546	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4547	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4548
4549	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4550	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4551	}
4552	else
4553	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4554
4555	if (fOffsettedTsc)
4556	{
4557	uint64_t u64CurTSC = ASMReadTSC();
4558	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4559	{
4560	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4561	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4562
4563	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4564	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4565	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4566	}
4567	else
4568	{
4569	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4570	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4571	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4572	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4573	}
4574	}
4575	else
4576	{
4577	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4578	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4579	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4580	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4581	}
4582	}
4583
4584
4585	/**
4586	* Determines if an exception is a contributory exception. Contributory
4587	* exceptions are ones which can cause double-faults.
4588	*
4589	* @returns true if the exception is contributory, false otherwise.
4590	* @param uVector The exception vector.
4591	*/
4592	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4593	{
4594	switch (uVector)
4595	{
4596	case X86_XCPT_GP:
4597	case X86_XCPT_SS:
4598	case X86_XCPT_NP:
4599	case X86_XCPT_TS:
4600	case X86_XCPT_DE:
4601	return true;
4602	default:
4603	break;
4604	}
4605	return false;
4606	}
4607
4608
4609	/**
4610	* Determines if an exception is a benign exception. Benign exceptions
4611	* are ones which cannot cause double-faults.
4612	*
4613	* @returns true if the exception is benign, false otherwise.
4614	* @param uVector The exception vector.
4615	*/
4616	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4617	{
4618	if ( uVector == X86_XCPT_PF
4619	\|\| uVector == X86_XCPT_DF)
4620	{
4621	return false;
4622	}
4623
4624	bool fIsBenignXcpt = !hmR0VmxIsContributoryXcpt(uVector);
4625	#ifdef VBOX_STRICT
4626	switch (uVector)
4627	{
4628	case X86_XCPT_DB:
4629	case X86_XCPT_NMI:
4630	case X86_XCPT_BP:
4631	case X86_XCPT_OF:
4632	case X86_XCPT_BR:
4633	case X86_XCPT_UD:
4634	case X86_XCPT_NM:
4635	case X86_XCPT_CO_SEG_OVERRUN:
4636	case X86_XCPT_MF:
4637	case X86_XCPT_AC:
4638	case X86_XCPT_MC:
4639	case X86_XCPT_XF:
4640	AssertMsg(fIsBenignXcpt, ("%#x\n", uVector));
4641	break;
4642	default:
4643	AssertMsg(!fIsBenignXcpt, ("%#x\n", uVector));
4644	break;
4645	}
4646	#endif
4647	return fIsBenignXcpt;
4648	}
4649
4650
4651	/**
4652	* Sets an event as a pending event to be injected into the guest.
4653	*
4654	* @param pVCpu Pointer to the VMCPU.
4655	* @param u32IntrInfo The VM-entry interruption-information field.
4656	* @param cbInstr The VM-entry instruction length in bytes (for software
4657	* interrupts, exceptions and privileged software
4658	* exceptions).
4659	* @param u32ErrCode The VM-entry exception error code.
4660	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4661	* page-fault.
4662	*/
4663	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4664	RTGCUINTPTR GCPtrFaultAddress)
4665	{
4666	Assert(!pVCpu->hm.s.Event.fPending);
4667	pVCpu->hm.s.Event.fPending = true;
4668	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4669	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4670	pVCpu->hm.s.Event.cbInstr = cbInstr;
4671	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4672	}
4673
4674
4675	/**
4676	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4677	*
4678	* @param pVCpu Pointer to the VMCPU.
4679	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4680	* out-of-sync. Make sure to update the required fields
4681	* before using them.
4682	*/
4683	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4684	{
4685	/* Inject the double-fault. */
4686	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4687	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4688	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4689	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4690	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4691	}
4692
4693
4694	/**
4695	* Handle a condition that occurred while delivering an event through the guest
4696	* IDT.
4697	*
4698	* @returns VBox status code (informational error codes included).
4699	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4700	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4701	* continue execution of the guest which will delivery the #DF.
4702	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4703	*
4704	* @param pVCpu Pointer to the VMCPU.
4705	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4706	* out-of-sync. Make sure to update the required fields
4707	* before using them.
4708	* @param pVmxTransient Pointer to the VMX transient structure.
4709	*
4710	* @remarks No-long-jump zone!!!
4711	*/
4712	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4713	{
4714	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4715	AssertRC(rc);
4716	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4717	{
4718	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4719	AssertRCReturn(rc, rc);
4720
4721	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4722	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4723	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4724
4725	typedef enum
4726	{
4727	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4728	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4729	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4730	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4731	} VMXREFLECTXCPT;
4732
4733	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4734	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4735	RTGCUINTPTR GCPtrFaultAddress = 0;
4736	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4737	{
4738	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4739	\|\| hmR0VmxIsBenignXcpt(uExitVector))
4740	{
4741	enmReflect = VMXREFLECTXCPT_XCPT;
4742	}
4743	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4744	&& uExitVector == X86_XCPT_PF)
4745	{
4746	enmReflect = VMXREFLECTXCPT_XCPT;
4747	GCPtrFaultAddress = pMixedCtx->cr2;
4748	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4749	}
4750	else if ( uExitVector == X86_XCPT_PF
4751	&& uIdtVector == X86_XCPT_PF)
4752	{
4753	pVmxTransient->fVectoringPF = true;
4754	enmReflect = VMXREFLECTXCPT_XCPT;
4755	GCPtrFaultAddress = pMixedCtx->cr2;
4756	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4757	}
4758	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & VMX_CONTRIBUTORY_XCPT_BITMAP)
4759	&& hmR0VmxIsContributoryXcpt(uExitVector)
4760	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4761	\|\| uIdtVector == X86_XCPT_PF))
4762	{
4763	enmReflect = VMXREFLECTXCPT_DF;
4764	}
4765	else if (uIdtVector == X86_XCPT_DF)
4766	enmReflect = VMXREFLECTXCPT_TF;
4767	else
4768	AssertMsgFailed(("Invalid!\n"));
4769	}
4770	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4771	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4772	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4773	{
4774	/*
4775	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4776	* (whatever they are) as they reoccur when restarting the instruction.
4777	*/
4778	enmReflect = VMXREFLECTXCPT_XCPT;
4779	}
4780
4781	switch (enmReflect)
4782	{
4783	case VMXREFLECTXCPT_XCPT:
4784	{
4785	uint32_t u32ErrCode = 0;
4786	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4787	{
4788	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4789	AssertRCReturn(rc, rc);
4790	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4791	}
4792	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4793	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4794	rc = VINF_SUCCESS;
4795	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4796	break;
4797	}
4798
4799	case VMXREFLECTXCPT_DF:
4800	{
4801	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4802	rc = VINF_VMX_DOUBLE_FAULT;
4803	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4804	uExitVector));
4805	break;
4806	}
4807
4808	case VMXREFLECTXCPT_TF:
4809	{
4810	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4811	rc = VINF_EM_RESET;
4812	break;
4813	}
4814
4815	default:
4816	Assert(rc == VINF_SUCCESS);
4817	break;
4818	}
4819	}
4820	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4821	return rc;
4822	}
4823
4824
4825	/**
4826	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4827	*
4828	* @returns VBox status code.
4829	* @param pVCpu Pointer to the VMCPU.
4830	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4831	* out-of-sync. Make sure to update the required fields
4832	* before using them.
4833	*
4834	* @remarks No-long-jump zone!!!
4835	*/
4836	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837	{
4838	int rc = VINF_SUCCESS;
4839	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4840	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4841	{
4842	RTGCUINTREG uVal = 0;
4843	RTCCUINTREG uShadow = 0;
4844	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4845	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4846	AssertRCReturn(rc, rc);
4847	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4848	CPUMSetGuestCR0(pVCpu, uVal);
4849	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4850	}
4851	return rc;
4852	}
4853
4854
4855	/**
4856	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4857	*
4858	* @returns VBox status code.
4859	* @param pVCpu Pointer to the VMCPU.
4860	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4861	* out-of-sync. Make sure to update the required fields
4862	* before using them.
4863	*
4864	* @remarks No-long-jump zone!!!
4865	*/
4866	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4867	{
4868	int rc = VINF_SUCCESS;
4869	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4870	{
4871	RTGCUINTREG uVal = 0;
4872	RTCCUINTREG uShadow = 0;
4873	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4874	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4875	AssertRCReturn(rc, rc);
4876	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4877	CPUMSetGuestCR4(pVCpu, uVal);
4878	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4879	}
4880	return rc;
4881	}
4882
4883
4884	/**
4885	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4886	*
4887	* @returns VBox status code.
4888	* @param pVCpu Pointer to the VMCPU.
4889	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4890	* out-of-sync. Make sure to update the required fields
4891	* before using them.
4892	*
4893	* @remarks No-long-jump zone!!!
4894	*/
4895	DECLINLINE(int) hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4896	{
4897	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4898	return VINF_SUCCESS;
4899
4900	RTGCUINTREG uVal = 0;
4901	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4902	AssertRCReturn(rc, rc);
4903	pMixedCtx->rip = uVal;
4904	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4905	return rc;
4906	}
4907
4908
4909	/**
4910	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4911	*
4912	* @returns VBox status code.
4913	* @param pVCpu Pointer to the VMCPU.
4914	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4915	* out-of-sync. Make sure to update the required fields
4916	* before using them.
4917	*
4918	* @remarks No-long-jump zone!!!
4919	*/
4920	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4921	{
4922	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4923	return VINF_SUCCESS;
4924
4925	RTGCUINTREG uVal = 0;
4926	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4927	AssertRCReturn(rc, rc);
4928	pMixedCtx->rsp = uVal;
4929	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4930	return rc;
4931	}
4932
4933
4934	/**
4935	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4936	*
4937	* @returns VBox status code.
4938	* @param pVCpu Pointer to the VMCPU.
4939	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4940	* out-of-sync. Make sure to update the required fields
4941	* before using them.
4942	*
4943	* @remarks No-long-jump zone!!!
4944	*/
4945	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4946	{
4947	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4948	return VINF_SUCCESS;
4949
4950	RTGCUINTREG uVal = 0;
4951	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4952	AssertRCReturn(rc, rc);
4953	pMixedCtx->rflags.u64 = uVal;
4954
4955	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4956	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4957	{
4958	PVM pVM = pVCpu->CTX_SUFF(pVM);
4959	Assert(pVM->hm.s.vmx.pRealModeTSS);
4960	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4961	pMixedCtx->eflags.Bits.u1VM = 0;
4962	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4963	}
4964
4965	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4966	return rc;
4967	}
4968
4969
4970	/**
4971	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4972	* guest-CPU context.
4973	*/
4974	static int hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4975	{
4976	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4977	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4978	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4979	return rc;
4980	}
4981
4982
4983	/**
4984	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4985	* from the guest-state area in the VMCS.
4986	*
4987	* @param pVCpu Pointer to the VMCPU.
4988	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4989	* out-of-sync. Make sure to update the required fields
4990	* before using them.
4991	*
4992	* @remarks No-long-jump zone!!!
4993	*/
4994	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4995	{
4996	uint32_t uIntrState = 0;
4997	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4998	AssertRC(rc);
4999
5000	if (!uIntrState)
5001	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5002	else
5003	{
5004	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5005	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5006	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5007	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5008	AssertRC(rc);
5009	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5010	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5011	}
5012	}
5013
5014
5015	/**
5016	* Saves the guest's activity state.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5029	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
5030	return VINF_SUCCESS;
5031	}
5032
5033
5034	/**
5035	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5036	* the current VMCS into the guest-CPU context.
5037	*
5038	* @returns VBox status code.
5039	* @param pVCpu Pointer to the VMCPU.
5040	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5041	* out-of-sync. Make sure to update the required fields
5042	* before using them.
5043	*
5044	* @remarks No-long-jump zone!!!
5045	*/
5046	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5047	{
5048	int rc = VINF_SUCCESS;
5049	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5050	{
5051	uint32_t u32Val = 0;
5052	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5053	pMixedCtx->SysEnter.cs = u32Val;
5054	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5055	}
5056
5057	RTGCUINTREG uGCVal = 0;
5058	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5059	{
5060	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5061	pMixedCtx->SysEnter.eip = uGCVal;
5062	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5063	}
5064	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5065	{
5066	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5067	pMixedCtx->SysEnter.esp = uGCVal;
5068	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5069	}
5070	return rc;
5071	}
5072
5073
5074	/**
5075	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5076	* context.
5077	*
5078	* @returns VBox status code.
5079	* @param pVCpu Pointer to the VMCPU.
5080	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5081	* out-of-sync. Make sure to update the required fields
5082	* before using them.
5083	*
5084	* @remarks No-long-jump zone!!!
5085	*/
5086	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5087	{
5088	RTGCUINTREG uVal = 0;
5089	int rc = VINF_SUCCESS;
5090	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5091	{
5092	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5093	pMixedCtx->fs.u64Base = uVal;
5094	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5095	}
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5102	* context.
5103	*
5104	* @returns VBox status code.
5105	* @param pVCpu Pointer to the VMCPU.
5106	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5107	* out-of-sync. Make sure to update the required fields
5108	* before using them.
5109	*
5110	* @remarks No-long-jump zone!!!
5111	*/
5112	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5113	{
5114	RTGCUINTREG uVal = 0;
5115	int rc = VINF_SUCCESS;
5116	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5117	{
5118	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5119	pMixedCtx->gs.u64Base = uVal;
5120	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5121	}
5122	return rc;
5123	}
5124
5125
5126	/**
5127	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5128	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5129	*
5130	* @returns VBox status code.
5131	* @param pVCpu Pointer to the VMCPU.
5132	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5133	* out-of-sync. Make sure to update the required fields
5134	* before using them.
5135	*
5136	* @remarks No-long-jump zone!!!
5137	*/
5138	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5139	{
5140	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5141	return VINF_SUCCESS;
5142
5143	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5144	{
5145	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5146	pMsr += i;
5147	switch (pMsr->u32IndexMSR)
5148	{
5149	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5150	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5151	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5152	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5153	#if 0
5154	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5155	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5156	#endif
5157	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5158	default:
5159	{
5160	AssertFailed();
5161	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5162	}
5163	}
5164	}
5165	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5166	return VINF_SUCCESS;
5167	}
5168
5169
5170	/**
5171	* Saves the guest control registers from the current VMCS into the guest-CPU
5172	* context.
5173	*
5174	* @returns VBox status code.
5175	* @param pVCpu Pointer to the VMCPU.
5176	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5177	* out-of-sync. Make sure to update the required fields
5178	* before using them.
5179	*
5180	* @remarks No-long-jump zone!!!
5181	*/
5182	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5183	{
5184	RTGCUINTREG uVal = 0;
5185	RTGCUINTREG uShadow = 0;
5186	int rc = VINF_SUCCESS;
5187
5188	/* Guest CR0. Guest FPU. */
5189	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5190
5191	/* Guest CR4. */
5192	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5193	AssertRCReturn(rc, rc);
5194
5195	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5196	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5197	{
5198	PVM pVM = pVCpu->CTX_SUFF(pVM);
5199	if ( pVM->hm.s.fNestedPaging
5200	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5201	{
5202	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5203	if (pMixedCtx->cr3 != uVal)
5204	{
5205	CPUMSetGuestCR3(pVCpu, uVal);
5206	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5207	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5208	}
5209
5210	/* We require EFER to check PAE mode. */
5211	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5212
5213	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5214	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5215	{
5216	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5217	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5218	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5219	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5220	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5221	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5222	}
5223	AssertRCReturn(rc, rc);
5224	}
5225
5226	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5227	}
5228	return rc;
5229	}
5230
5231
5232	/**
5233	* Reads a guest segment register from the current VMCS into the guest-CPU
5234	* context.
5235	*
5236	* @returns VBox status code.
5237	* @param pVCpu Pointer to the VMCPU.
5238	* @param idxSel Index of the selector in the VMCS.
5239	* @param idxLimit Index of the segment limit in the VMCS.
5240	* @param idxBase Index of the segment base in the VMCS.
5241	* @param idxAccess Index of the access rights of the segment in the VMCS.
5242	* @param pSelReg Pointer to the segment selector.
5243	*
5244	* @remarks No-long-jump zone!!!
5245	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5246	* as that takes care of whether to read from the VMCS cache or not.
5247	*/
5248	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5249	PCPUMSELREG pSelReg)
5250	{
5251	uint32_t u32Val = 0;
5252	int rc = VMXReadVmcs32(idxSel, &u32Val);
5253	pSelReg->Sel = (uint16_t)u32Val;
5254	pSelReg->ValidSel = (uint16_t)u32Val;
5255	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5256
5257	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5258	pSelReg->u32Limit = u32Val;
5259
5260	RTGCUINTREG uGCVal = 0;
5261	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5262	pSelReg->u64Base = uGCVal;
5263
5264	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5265	pSelReg->Attr.u = u32Val;
5266	AssertRCReturn(rc, rc);
5267
5268	/*
5269	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5270	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5271	*/
5272	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5273	{
5274	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5275	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5276	}
5277	return rc;
5278	}
5279
5280
5281	/**
5282	* Saves the guest segment registers from the current VMCS into the guest-CPU
5283	* context.
5284	*
5285	* @returns VBox status code.
5286	* @param pVCpu Pointer to the VMCPU.
5287	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5288	* out-of-sync. Make sure to update the required fields
5289	* before using them.
5290	*
5291	* @remarks No-long-jump zone!!!
5292	*/
5293	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5294	{
5295	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5296	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5297	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5298	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5299	#else
5300	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5301	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5302	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5303	#endif
5304
5305	int rc = VINF_SUCCESS;
5306
5307	/* Guest segment registers. */
5308	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5309	{
5310	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5311	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5312	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5313	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5314	rc \|= VMXLOCAL_READ_SEG(ES, es);
5315	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5316	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5317	AssertRCReturn(rc, rc);
5318
5319	/* Restore segment attributes for real-on-v86 mode hack. */
5320	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5321	{
5322	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5323	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5324	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5325	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5326	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5327	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5328	}
5329	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5330	}
5331
5332	/* Guest LDTR. */
5333	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5334	{
5335	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5336	AssertRCReturn(rc, rc);
5337	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5338	}
5339
5340	/* Guest GDTR. */
5341	RTGCUINTREG uGCVal = 0;
5342	uint32_t u32Val = 0;
5343	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5344	{
5345	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5346	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5347	pMixedCtx->gdtr.pGdt = uGCVal;
5348	pMixedCtx->gdtr.cbGdt = u32Val;
5349	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5350	}
5351
5352	/* Guest IDTR. */
5353	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5354	{
5355	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5356	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5357	pMixedCtx->idtr.pIdt = uGCVal;
5358	pMixedCtx->idtr.cbIdt = u32Val;
5359	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5360	}
5361
5362	/* Guest TR. */
5363	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5364	{
5365	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5366
5367	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5368	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5369	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5370	AssertRCReturn(rc, rc);
5371	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5372	}
5373	return rc;
5374	}
5375
5376
5377	/**
5378	* Saves the guest debug registers from the current VMCS into the guest-CPU
5379	* context.
5380	*
5381	* @returns VBox status code.
5382	* @param pVCpu Pointer to the VMCPU.
5383	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5384	* out-of-sync. Make sure to update the required fields
5385	* before using them.
5386	*
5387	* @remarks No-long-jump zone!!!
5388	*/
5389	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5390	{
5391	int rc = VINF_SUCCESS;
5392	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5393	{
5394	RTGCUINTREG uVal;
5395	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5396	pMixedCtx->dr[7] = uVal;
5397
5398	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5399	}
5400	return rc;
5401	}
5402
5403
5404	/**
5405	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5406	*
5407	* @returns VBox status code.
5408	* @param pVCpu Pointer to the VMCPU.
5409	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5410	* out-of-sync. Make sure to update the required fields
5411	* before using them.
5412	*
5413	* @remarks No-long-jump zone!!!
5414	*/
5415	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5416	{
5417	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5418	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5419	return VINF_SUCCESS;
5420	}
5421
5422
5423	/**
5424	* Saves the entire guest state from the currently active VMCS into the
5425	* guest-CPU context. This essentially VMREADs all guest-data.
5426	*
5427	* @returns VBox status code.
5428	* @param pVCpu Pointer to the VMCPU.
5429	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5430	* out-of-sync. Make sure to update the required fields
5431	* before using them.
5432	*/
5433	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5434	{
5435	Assert(pVCpu);
5436	Assert(pMixedCtx);
5437
5438	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5439	return VINF_SUCCESS;
5440
5441	VMMRZCallRing3Disable(pVCpu);
5442
5443	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5444	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5445
5446	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5447	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5448
5449	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5451
5452	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5454
5455	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5456	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5457
5458	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5459	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5460
5461	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5462	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5463
5464	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5465	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5466
5467	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5468	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5469
5470	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5471	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5472
5473	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5474	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5475
5476	VMMRZCallRing3Enable(pVCpu);
5477	return rc;
5478	}
5479
5480
5481	/**
5482	* Check per-VM and per-VCPU force flag actions that require us to go back to
5483	* ring-3 for one reason or another.
5484	*
5485	* @returns VBox status code (information status code included).
5486	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5487	* ring-3.
5488	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5489	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5490	* interrupts)
5491	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5492	* all EMTs to be in ring-3.
5493	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5494	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5495	* to the EM loop.
5496	*
5497	* @param pVM Pointer to the VM.
5498	* @param pVCpu Pointer to the VMCPU.
5499	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5500	* out-of-sync. Make sure to update the required fields
5501	* before using them.
5502	*/
5503	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5504	{
5505	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5506
5507	int rc = VERR_INTERNAL_ERROR_5;
5508	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5509	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5510	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5511	{
5512	/* We need the control registers now, make sure the guest-CPU context is updated. */
5513	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5514	AssertRCReturn(rc, rc);
5515
5516	/* Pending HM CR3 sync. */
5517	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5518	{
5519	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5520	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5521	}
5522	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5523	{
5524	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5525	AssertRC(rc);
5526	}
5527
5528	/* Pending PGM C3 sync. */
5529	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5530	{
5531	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5532	if (rc != VINF_SUCCESS)
5533	{
5534	AssertRC(rc);
5535	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5536	return rc;
5537	}
5538	}
5539
5540	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5541	/* -XXX- what was that about single stepping? */
5542	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5543	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5544	{
5545	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5546	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5547	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5548	return rc;
5549	}
5550
5551	/* Pending VM request packets, such as hardware interrupts. */
5552	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5553	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5554	{
5555	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5556	return VINF_EM_PENDING_REQUEST;
5557	}
5558
5559	/* Pending PGM pool flushes. */
5560	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5561	{
5562	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5563	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5564	}
5565
5566	/* Pending DMA requests. */
5567	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5568	{
5569	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5570	return VINF_EM_RAW_TO_R3;
5571	}
5572	}
5573
5574	/* Paranoia. */
5575	Assert(rc != VERR_EM_INTERPRETER);
5576	return VINF_SUCCESS;
5577	}
5578
5579
5580	/**
5581	* Converts any TRPM trap into a pending VMX event. This is typically used when
5582	* entering from ring-3 (not longjmp returns).
5583	*
5584	* @param pVCpu Pointer to the VMCPU.
5585	* @param pCtx Pointer to the guest-CPU context.
5586	*/
5587	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5588	{
5589	if (!TRPMHasTrap(pVCpu))
5590	{
5591	Assert(!pVCpu->hm.s.Event.fPending);
5592	return;
5593	}
5594
5595	uint8_t uVector;
5596	TRPMEVENT enmTrpmEvent;
5597	RTGCUINT uErrCode;
5598	RTGCUINTPTR GCPtrFaultAddress;
5599	uint8_t cbInstr;
5600
5601	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5602	AssertRC(rc);
5603
5604	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5605	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5606	uint32_t u32ErrCode = uErrCode;
5607	if (enmTrpmEvent == TRPM_TRAP)
5608	{
5609	switch (uVector)
5610	{
5611	case X86_XCPT_BP:
5612	case X86_XCPT_OF:
5613	{
5614	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5615	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5616	break;
5617	}
5618
5619	case X86_XCPT_PF:
5620	pCtx->cr2 = GCPtrFaultAddress;
5621	/* no break */
5622	case X86_XCPT_DF:
5623	case X86_XCPT_TS:
5624	case X86_XCPT_NP:
5625	case X86_XCPT_SS:
5626	case X86_XCPT_GP:
5627	case X86_XCPT_AC:
5628	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5629	them which VT-x/VMM pushes to the guest stack. */
5630	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5631	/* no break! */
5632	default:
5633	{
5634	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5635	break;
5636	}
5637	}
5638	}
5639	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5640	{
5641	if (uVector != X86_XCPT_NMI)
5642	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5643	else
5644	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5645	}
5646	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5647	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5648	else
5649	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5650
5651	rc = TRPMResetTrap(pVCpu);
5652	AssertRC(rc);
5653	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5654	u32IntrInfo, enmTrpmEvent, cbInstr, u32ErrCode, GCPtrFaultAddress));
5655	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5656	}
5657
5658
5659	/**
5660	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5661	* VT-x to execute any instruction.
5662	*
5663	* @param pvCpu Pointer to the VMCPU.
5664	*/
5665	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5666	{
5667	if (pVCpu->hm.s.Event.fPending)
5668	{
5669	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5670	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5671	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5672	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5673
5674	/* If a trap was already pending, we did something wrong! */
5675	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5676
5677	TRPMEVENT enmTrapType;
5678	switch (uVectorType)
5679	{
5680	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5681	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5682	enmTrapType = TRPM_HARDWARE_INT;
5683	break;
5684	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5685	enmTrapType = TRPM_SOFTWARE_INT;
5686	break;
5687	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5688	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5689	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5690	enmTrapType = TRPM_TRAP;
5691	break;
5692	default:
5693	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5694	enmTrapType = TRPM_32BIT_HACK;
5695	break;
5696	}
5697
5698	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5699	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5700	AssertRC(rc);
5701
5702	if (fErrorCodeValid)
5703	TRPMSetErrorCode(pVCpu, uErrorCode);
5704	/* A page-fault exception during a page-fault would become a double-fault. */
5705	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5706	&& uVector == X86_XCPT_PF)
5707	{
5708	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5709	}
5710	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5711	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5712	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5713	{
5714	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5715	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5716	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5717	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5718	}
5719	pVCpu->hm.s.Event.fPending = false;
5720	}
5721	}
5722
5723
5724	/**
5725	* Does the necessary state syncing before doing a longjmp to ring-3.
5726	*
5727	* @param pVM Pointer to the VM.
5728	* @param pVCpu Pointer to the VMCPU.
5729	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5730	* out-of-sync. Make sure to update the required fields
5731	* before using them.
5732	* @param rcExit The reason for exiting to ring-3. Can be
5733	* VINF_VMM_UNKNOWN_RING3_CALL.
5734	*
5735	* @remarks No-long-jmp zone!!!
5736	*/
5737	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5738	{
5739	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5740
5741	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5742	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5743	AssertRC(rc);
5744
5745	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5746	if (CPUMIsGuestFPUStateActive(pVCpu))
5747	{
5748	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5749	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5750	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5751	}
5752
5753	/* Restore debug registers if necessary and resync on next R0 reentry. */
5754	if (CPUMIsGuestDebugStateActive(pVCpu))
5755	{
5756	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5757	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5758	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5759	}
5760	else if (CPUMIsHyperDebugStateActive(pVCpu))
5761	{
5762	CPUMR0LoadHostDebugState(pVM, pVCpu);
5763	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5764	}
5765
5766	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5767	}
5768
5769
5770	/**
5771	* An action requires us to go back to ring-3. This function does the necessary
5772	* steps before we can safely return to ring-3. This is not the same as longjmps
5773	* to ring-3, this is voluntary.
5774	*
5775	* @param pVM Pointer to the VM.
5776	* @param pVCpu Pointer to the VMCPU.
5777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778	* out-of-sync. Make sure to update the required fields
5779	* before using them.
5780	* @param rcExit The reason for exiting to ring-3. Can be
5781	* VINF_VMM_UNKNOWN_RING3_CALL.
5782	*/
5783	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5784	{
5785	Assert(pVM);
5786	Assert(pVCpu);
5787	Assert(pMixedCtx);
5788	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5789
5790	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5791	{
5792	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5793	return;
5794	}
5795	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5796	{
5797	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5798	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5799	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5800	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5801	return;
5802	}
5803
5804	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5805	VMMRZCallRing3Disable(pVCpu);
5806	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5807
5808	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5809	hmR0VmxUpdateTRPM(pVCpu);
5810
5811	/* Sync. the guest state. */
5812	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5813	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5814
5815	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5816	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5817	\| CPUM_CHANGED_LDTR
5818	\| CPUM_CHANGED_GDTR
5819	\| CPUM_CHANGED_IDTR
5820	\| CPUM_CHANGED_TR
5821	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5822
5823	/* On our way back from ring-3 the following needs to be done. */
5824	/** @todo This can change with preemption hooks. */
5825	if (rcExit == VINF_EM_RAW_INTERRUPT)
5826	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5827	else
5828	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5829
5830	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5831	VMMRZCallRing3Enable(pVCpu);
5832	}
5833
5834
5835	/**
5836	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5837	* longjump to ring-3 and possibly get preempted.
5838	*
5839	* @param pVCpu Pointer to the VMCPU.
5840	* @param enmOperation The operation causing the ring-3 longjump.
5841	* @param pvUser The user argument (pointer to the possibly
5842	* out-of-date guest-CPU context).
5843	*
5844	* @remarks Must never be called with @a enmOperation ==
5845	* VMMCALLRING3_VM_R0_ASSERTION.
5846	*/
5847	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5848	{
5849	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5850	Assert(pVCpu);
5851	Assert(pvUser);
5852	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5853	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5854
5855	VMMRZCallRing3Disable(pVCpu);
5856	Log(("hmR0VmxLongJmpToRing3\n"));
5857	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5858	VMMRZCallRing3Enable(pVCpu);
5859	}
5860
5861
5862	/**
5863	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5864	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5865	*
5866	* @returns VBox status code.
5867	* @param pVCpu Pointer to the VMCPU.
5868	*/
5869	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5870	{
5871	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5872	{
5873	/* Enable interrupt-window exiting if it's not in effect already. */
5874	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5875	{
5876	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5877	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5878	AssertRC(rc);
5879	}
5880	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5881	}
5882
5883
5884	/**
5885	* Injects any pending events into the guest if the guest is in a state to
5886	* receive them.
5887	*
5888	* @returns VBox status code (informational status codes included).
5889	* @param pVCpu Pointer to the VMCPU.
5890	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5891	* out-of-sync. Make sure to update the required fields
5892	* before using them.
5893	*/
5894	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5895	{
5896	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5897	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5898	const bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5899	const bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5900
5901	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS));
5902	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5903	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5904	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5905
5906	int rc = VINF_SUCCESS;
5907	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5908	{
5909	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5910	bool fInject = true;
5911	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5912	{
5913	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5914	AssertRCReturn(rc, rc);
5915	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5916	if ( fBlockInt
5917	\|\| fBlockSti
5918	\|\| fBlockMovSS)
5919	{
5920	fInject = false;
5921	}
5922	}
5923	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5924	&& ( fBlockMovSS
5925	\|\| fBlockSti))
5926	{
5927	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5928	fInject = false;
5929	}
5930
5931	if (fInject)
5932	{
5933	Log(("Injecting pending event\n"));
5934	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5935	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5936	AssertRCReturn(rc, rc);
5937	pVCpu->hm.s.Event.fPending = false;
5938	}
5939	else
5940	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5941	} /** @todo SMI. SMIs take priority over NMIs. */
5942	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5943	{
5944	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5945	if ( !fBlockMovSS
5946	&& !fBlockSti)
5947	{
5948	Log(("Injecting NMI\n"));
5949	RTGCUINTPTR uIntrInfo;
5950	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5951	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5952	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5953	AssertRCReturn(rc, rc);
5954	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5955	}
5956	else
5957	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5958	}
5959	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5960	{
5961	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5962	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5963	AssertRCReturn(rc, rc);
5964	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5965	if ( !fBlockInt
5966	&& !fBlockSti
5967	&& !fBlockMovSS)
5968	{
5969	uint8_t u8Interrupt;
5970	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5971	if (RT_SUCCESS(rc))
5972	{
5973	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5974	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5975	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5976	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5977	}
5978	else
5979	{
5980	/** @todo Does this actually happen? If not turn it into an assertion. */
5981	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5982	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5983	rc = VINF_SUCCESS;
5984	}
5985	}
5986	else
5987	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5988	}
5989
5990	/*
5991	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5992	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5993	*/
5994	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5995	AssertRC(rc2);
5996	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5997	return rc;
5998	}
5999
6000
6001	/**
6002	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6003	*
6004	* @param pVCpu Pointer to the VMCPU.
6005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6006	* out-of-sync. Make sure to update the required fields
6007	* before using them.
6008	*/
6009	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6010	{
6011	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6012	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6013	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6014	}
6015
6016
6017	/**
6018	* Injects a double-fault (#DF) exception into the VM.
6019	*
6020	* @returns VBox status code (informational status code included).
6021	* @param pVCpu Pointer to the VMCPU.
6022	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6023	* out-of-sync. Make sure to update the required fields
6024	* before using them.
6025	*/
6026	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6027	{
6028	/* Inject the double-fault. */
6029	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6030	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6031	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6032	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6033	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6034	}
6035
6036
6037	/**
6038	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6039	*
6040	* @param pVCpu Pointer to the VMCPU.
6041	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6042	* out-of-sync. Make sure to update the required fields
6043	* before using them.
6044	*/
6045	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6046	{
6047	/* Inject the debug-exception. */
6048	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6049	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6050	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6051	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6052	}
6053
6054
6055	/**
6056	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6057	*
6058	* @param pVCpu Pointer to the VMCPU.
6059	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6060	* out-of-sync. Make sure to update the required fields
6061	* before using them.
6062	* @param cbInstr The value of RIP that is to be pushed on the guest
6063	* stack.
6064	*/
6065	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6066	{
6067	/* Inject the overflow exception. */
6068	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6069	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6070	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6071	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6072	}
6073
6074
6075	/**
6076	* Injects a general-protection (#GP) fault into the VM.
6077	*
6078	* @returns VBox status code (informational status code included).
6079	* @param pVCpu Pointer to the VMCPU.
6080	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6081	* out-of-sync. Make sure to update the required fields
6082	* before using them.
6083	* @param u32ErrorCode The error code associated with the #GP.
6084	*/
6085	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6086	uint32_t *puIntrState)
6087	{
6088	/* Inject the general-protection fault. */
6089	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6090	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6091	if (fErrorCodeValid)
6092	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6093	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6094	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6095	}
6096
6097
6098	/**
6099	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6100	*
6101	* @param pVCpu Pointer to the VMCPU.
6102	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6103	* out-of-sync. Make sure to update the required fields
6104	* before using them.
6105	* @param uVector The software interrupt vector number.
6106	* @param cbInstr The value of RIP that is to be pushed on the guest
6107	* stack.
6108	*/
6109	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6110	{
6111	/* Inject the INTn. */
6112	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6113	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6114	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6115	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6116	}
6117
6118
6119	/**
6120	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6121	* stack.
6122	*
6123	* @returns VBox status code (information status code included).
6124	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6125	* @param pVM Pointer to the VM.
6126	* @param pMixedCtx Pointer to the guest-CPU context.
6127	* @param uValue The value to push to the guest stack.
6128	*/
6129	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6130	{
6131	/*
6132	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6133	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6134	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6135	*/
6136	if (pMixedCtx->sp == 1)
6137	return VINF_EM_RESET;
6138	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6139	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6140	AssertRCReturn(rc, rc);
6141	return rc;
6142	}
6143
6144
6145	/**
6146	* Injects an event into the guest upon VM-entry by updating the relevant fields
6147	* in the VM-entry area in the VMCS.
6148	*
6149	* @returns VBox status code (informational error codes included).
6150	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6151	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6152	*
6153	* @param pVCpu Pointer to the VMCPU.
6154	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6155	* out-of-sync. Make sure to update the required fields
6156	* before using them.
6157	* @param u64IntrInfo The VM-entry interruption-information field.
6158	* @param cbInstr The VM-entry instruction length in bytes (for software
6159	* interrupts, exceptions and privileged software
6160	* exceptions).
6161	* @param u32ErrCode The VM-entry exception error code.
6162	* @param puIntrState Pointer to the current guest interruptibility-state.
6163	* This interruptibility-state will be updated if
6164	* necessary. This cannot not be NULL.
6165	*
6166	* @remarks No-long-jump zone!!!
6167	* @remarks Requires CR0!
6168	*/
6169	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6170	uint32_t u32ErrCode, uint32_t *puIntrState)
6171	{
6172	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6173	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6174	Assert(puIntrState);
6175	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6176
6177	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6178	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6179
6180	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6181	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6182	\|\| !((*puIntrState) & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6183
6184	/* We require CR0 to check if the guest is in real-mode. */
6185	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6186	AssertRCReturn(rc, rc);
6187
6188	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6189
6190	/*
6191	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6192	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6193	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6194	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6195	*/
6196	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6197	{
6198	PVM pVM = pVCpu->CTX_SUFF(pVM);
6199	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6200	{
6201	Assert(PDMVmmDevHeapIsEnabled(pVM));
6202	Assert(pVM->hm.s.vmx.pRealModeTSS);
6203
6204	/* Save the required guest state bits from the VMCS. */
6205	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6206	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6207	AssertRCReturn(rc, rc);
6208
6209	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6210	const size_t cbIdtEntry = 4;
6211	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6212	{
6213	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6214	if (uVector == X86_XCPT_DF)
6215	return VINF_EM_RESET;
6216	else if (uVector == X86_XCPT_GP)
6217	{
6218	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6219	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6220	}
6221
6222	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6223	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6224	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6225	}
6226
6227	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6228	uint16_t uGuestIp = pMixedCtx->ip;
6229	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6230	{
6231	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6232	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6233	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6234	}
6235	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6236	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6237
6238	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6239	uint16_t offIdtEntry = 0;
6240	RTSEL selIdtEntry = 0;
6241	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6242	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6243	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6244	AssertRCReturn(rc, rc);
6245
6246	/* Construct the stack frame for the interrupt/exception handler. */
6247	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6248	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6249	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6250	AssertRCReturn(rc, rc);
6251
6252	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6253	if (rc == VINF_SUCCESS)
6254	{
6255	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6256	pMixedCtx->rip = offIdtEntry;
6257	pMixedCtx->cs.Sel = selIdtEntry;
6258	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6259	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6260	\| HM_CHANGED_GUEST_RIP
6261	\| HM_CHANGED_GUEST_RFLAGS
6262	\| HM_CHANGED_GUEST_RSP;
6263
6264	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6265	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6266	{
6267	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6268	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6269	Log(("Clearing inhibition due to STI.\n"));
6270	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6271	}
6272	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6273	}
6274	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6275	return rc;
6276	}
6277	else
6278	{
6279	/*
6280	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6281	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6282	*/
6283	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6284	}
6285	}
6286
6287	/* Validate. */
6288	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6289	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6290	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6291
6292	/* Inject. */
6293	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6294	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6295	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6296	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6297	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6298	AssertRCReturn(rc, rc);
6299	return rc;
6300	}
6301
6302
6303	/**
6304	* Enters the VT-x session.
6305	*
6306	* @returns VBox status code.
6307	* @param pVM Pointer to the VM.
6308	* @param pVCpu Pointer to the VMCPU.
6309	* @param pCpu Pointer to the CPU info struct.
6310	*/
6311	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6312	{
6313	AssertPtr(pVM);
6314	AssertPtr(pVCpu);
6315	Assert(pVM->hm.s.vmx.fSupported);
6316	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6317	NOREF(pCpu);
6318
6319	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6320
6321	/* Make sure we're in VMX root mode. */
6322	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6323	if (!(u32HostCR4 & X86_CR4_VMXE))
6324	{
6325	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6326	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6327	}
6328
6329	/* Load the active VMCS as the current one. */
6330	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6331	if (RT_FAILURE(rc))
6332	return rc;
6333
6334	/** @todo this will change with preemption hooks where can can VMRESUME as long
6335	* as we're no preempted. */
6336	pVCpu->hm.s.fResumeVM = false;
6337	return VINF_SUCCESS;
6338	}
6339
6340
6341	/**
6342	* Leaves the VT-x session.
6343	*
6344	* @returns VBox status code.
6345	* @param pVM Pointer to the VM.
6346	* @param pVCpu Pointer to the VMCPU.
6347	* @param pCtx Pointer to the guest-CPU context.
6348	*/
6349	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6350	{
6351	AssertPtr(pVCpu);
6352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6353	NOREF(pVM);
6354	NOREF(pCtx);
6355
6356	/** @todo this will change with preemption hooks where we only VMCLEAR when
6357	* we are actually going to be preempted, not all the time like we
6358	* currently do. */
6359	/*
6360	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6361	* and mark the VMCS launch-state as "clear".
6362	*/
6363	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6364	return rc;
6365	}
6366
6367
6368	/**
6369	* Saves the host state in the VMCS host-state.
6370	* Sets up the VM-exit MSR-load area.
6371	*
6372	* The CPU state will be loaded from these fields on every successful VM-exit.
6373	*
6374	* @returns VBox status code.
6375	* @param pVM Pointer to the VM.
6376	* @param pVCpu Pointer to the VMCPU.
6377	*
6378	* @remarks No-long-jump zone!!!
6379	*/
6380	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6381	{
6382	AssertPtr(pVM);
6383	AssertPtr(pVCpu);
6384	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6385
6386	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6387
6388	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6389	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6390	return VINF_SUCCESS;
6391
6392	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6393	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6394
6395	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6396	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6397
6398	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6399	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6400
6401	/* Reset the host-state-changed flag. */
6402	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6403	return rc;
6404	}
6405
6406
6407	/**
6408	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6409	* loaded from these fields on every successful VM-entry.
6410	*
6411	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6412	* Sets up the VM-entry controls.
6413	* Sets up the appropriate VMX non-root function to execute guest code based on
6414	* the guest CPU mode.
6415	*
6416	* @returns VBox status code.
6417	* @param pVM Pointer to the VM.
6418	* @param pVCpu Pointer to the VMCPU.
6419	* @param pCtx Pointer to the guest-CPU context.
6420	*
6421	* @remarks No-long-jump zone!!!
6422	*/
6423	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6424	{
6425	AssertPtr(pVM);
6426	AssertPtr(pVCpu);
6427	AssertPtr(pCtx);
6428	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6429
6430	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6431
6432	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6433
6434	/* Determine real-on-v86 mode. */
6435	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6436	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6437	&& CPUMIsGuestInRealModeEx(pCtx))
6438	{
6439	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6440	}
6441
6442	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6443	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6444
6445	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6446	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6447
6448	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6449	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6450
6451	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6452	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6453
6454	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6455	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6456
6457	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6458	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6459
6460	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6462
6463	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6465
6466	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6467	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6468
6469	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6470	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6471
6472	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6473	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6474	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6475
6476	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6477	return rc;
6478	}
6479
6480
6481	/**
6482	* Does the preparations before executing guest code in VT-x.
6483	*
6484	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6485	* recompiler. We must be cautious what we do here regarding committing
6486	* guest-state information into the the VMCS assuming we assuredly execute the
6487	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6488	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6489	* that the recompiler can (and should) use them when it resumes guest
6490	* execution. Otherwise such operations must be done when we can no longer
6491	* exit to ring-3.
6492	*
6493	* @returns VBox status code (informational status codes included).
6494	* @retval VINF_SUCCESS if we can proceed with running the guest.
6495	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6496	* into the guest.
6497	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6498	*
6499	* @param pVM Pointer to the VM.
6500	* @param pVCpu Pointer to the VMCPU.
6501	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6502	* out-of-sync. Make sure to update the required fields
6503	* before using them.
6504	* @param pVmxTransient Pointer to the VMX transient structure.
6505	*
6506	* @remarks Called with preemption disabled.
6507	*/
6508	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6509	{
6510	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6511
6512	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6513	PGMRZDynMapFlushAutoSet(pVCpu);
6514	#endif
6515
6516	/* Check force flag actions that might require us to go back to ring-3. */
6517	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6518	if (rc != VINF_SUCCESS)
6519	return rc;
6520
6521	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6522	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6523	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6524	{
6525	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6526	RTGCPHYS GCPhysApicBase;
6527	GCPhysApicBase = pMixedCtx->msrApicBase;
6528	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6529
6530	/* Unalias any existing mapping. */
6531	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6532	AssertRCReturn(rc, rc);
6533
6534	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6535	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6536	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6537	AssertRCReturn(rc, rc);
6538
6539	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6540	}
6541
6542	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6543	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6544	pVmxTransient->uEFlags = ASMIntDisableFlags();
6545	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6546	{
6547	ASMSetFlags(pVmxTransient->uEFlags);
6548	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6549	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6550	return VINF_EM_RAW_INTERRUPT;
6551	}
6552	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6553	#endif
6554
6555	/*
6556	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6557	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6558	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6559	*/
6560	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6561	AssertRCReturn(rc, rc);
6562	return rc;
6563	}
6564
6565
6566	/**
6567	* Prepares to run guest code in VT-x and we've committed to doing so. This
6568	* means there is no backing out to ring-3 or anywhere else at this
6569	* point.
6570	*
6571	* @param pVM Pointer to the VM.
6572	* @param pVCpu Pointer to the VMCPU.
6573	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6574	* out-of-sync. Make sure to update the required fields
6575	* before using them.
6576	* @param pVmxTransient Pointer to the VMX transient structure.
6577	*
6578	* @remarks Called with preemption disabled.
6579	* @remarks No-long-jump zone!!!
6580	*/
6581	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6582	{
6583	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6584	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6585
6586	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6587	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6588	pVmxTransient->uEFlags = ASMIntDisableFlags();
6589	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6590	#endif
6591
6592	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6593	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6594	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6595	int rc = VINF_SUCCESS;
6596	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6597	{
6598	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6599	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6600	}
6601	else if (pVCpu->hm.s.fContextUseFlags)
6602	{
6603	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6604	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6605	}
6606	AssertRC(rc);
6607	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6608
6609	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6610	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6611	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6612
6613	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6614	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6615
6616	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6617	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6618	{
6619	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6620	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6621	}
6622
6623	/*
6624	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6625	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6626	*/
6627	if (pVM->hm.s.fTPRPatchingActive)
6628	{
6629	Assert(!CPUMIsGuestInLongMode(pVCpu));
6630
6631	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6632	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6633	AssertRC(rc);
6634
6635	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6636	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6637	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6638	see hmR0VmxLoadGuestApicState(). */
6639	}
6640
6641	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6642	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6643	to start executing. */
6644	}
6645
6646
6647	/**
6648	* Performs some essential restoration of state after running guest code in
6649	* VT-x.
6650	*
6651	* @param pVM Pointer to the VM.
6652	* @param pVCpu Pointer to the VMCPU.
6653	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6654	* out-of-sync. Make sure to update the required fields
6655	* before using them.
6656	* @param pVmxTransient Pointer to the VMX transient structure.
6657	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6658	*
6659	* @remarks Called with interrupts disabled.
6660	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6661	* unconditionally when it is safe to do so.
6662	*/
6663	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6664	{
6665	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6666	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6667
6668	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6669	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6670	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6671	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6672	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6673
6674	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6675	{
6676	/** @todo Find a way to fix hardcoding a guestimate. */
6677	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6678	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6679	}
6680
6681	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6682	Assert(!(ASMGetFlags() & X86_EFL_IF));
6683	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6684
6685	/* Restore the effects of TPR patching if any. */
6686	if (pVM->hm.s.fTPRPatchingActive)
6687	{
6688	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6689	AssertRC(rc);
6690	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6691	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6692	}
6693
6694	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6695	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6696
6697	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6698	uint32_t uExitReason;
6699	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6700	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6701	AssertRC(rc);
6702	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6703	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6704
6705	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6706	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6707
6708	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6709	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6710	{
6711	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6712	return;
6713	}
6714
6715	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6716	{
6717	/* Update the guest interruptibility-state from the VMCS. */
6718	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6719
6720	/*
6721	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6722	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6723	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6724	*/
6725	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6726	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6727	{
6728	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6729	AssertRC(rc);
6730	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6731	}
6732	}
6733	}
6734
6735
6736	/**
6737	* Runs the guest code using VT-x.
6738	*
6739	* @returns VBox status code.
6740	* @param pVM Pointer to the VM.
6741	* @param pVCpu Pointer to the VMCPU.
6742	* @param pCtx Pointer to the guest-CPU context.
6743	*
6744	* @remarks Called with preemption disabled.
6745	*/
6746	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6747	{
6748	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6749	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6750
6751	VMXTRANSIENT VmxTransient;
6752	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6753	int rc = VERR_INTERNAL_ERROR_5;
6754	uint32_t cLoops = 0;
6755	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6756
6757	for (;; cLoops++)
6758	{
6759	Assert(!HMR0SuspendPending());
6760	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6761	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6762	(unsigned)RTMpCpuId(), cLoops));
6763
6764	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6765	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6766	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6767	if (rc != VINF_SUCCESS)
6768	break;
6769
6770	/*
6771	* No longjmps to ring-3 from this point on!!!
6772	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6773	* This also disables flushing of the R0-logger instance (if any).
6774	*/
6775	VMMRZCallRing3Disable(pVCpu);
6776	VMMRZCallRing3RemoveNotification(pVCpu);
6777	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6778
6779	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6780	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6781
6782	/*
6783	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6784	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6785	*/
6786	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6787	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6788	{
6789	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6790	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6791	return rc;
6792	}
6793
6794	/* Handle the VM-exit. */
6795	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6796	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6797	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6798	rc = (*g_apfnVMExitHandlers[VmxTransient.uExitReason])(pVCpu, pCtx, &VmxTransient);
6799	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6800	if (rc != VINF_SUCCESS)
6801	break;
6802	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6803	{
6804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6805	rc = VINF_EM_RAW_INTERRUPT;
6806	break;
6807	}
6808	}
6809
6810	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6811	if (rc == VERR_EM_INTERPRETER)
6812	rc = VINF_EM_RAW_EMULATE_INSTR;
6813	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6814	return rc;
6815	}
6816
6817	#if 1
6818	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6819	{
6820	int rc;
6821	switch (rcReason)
6822	{
6823	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6856
6857	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6862	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6863	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6864	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6865	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6866
6867	case VMX_EXIT_VMCALL:
6868	case VMX_EXIT_VMCLEAR:
6869	case VMX_EXIT_VMLAUNCH:
6870	case VMX_EXIT_VMPTRLD:
6871	case VMX_EXIT_VMPTRST:
6872	case VMX_EXIT_VMREAD:
6873	case VMX_EXIT_VMRESUME:
6874	case VMX_EXIT_VMWRITE:
6875	case VMX_EXIT_VMXOFF:
6876	case VMX_EXIT_VMXON:
6877	case VMX_EXIT_INVEPT:
6878	case VMX_EXIT_INVVPID:
6879	case VMX_EXIT_VMFUNC:
6880	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6881	break;
6882	default:
6883	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6884	break;
6885	}
6886	return rc;
6887	}
6888	#endif
6889
6890	#ifdef DEBUG
6891	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6892	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6893	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6894	# define VMX_ASSERT_PREEMPT_CPUID() \
6895	do \
6896	{ \
6897	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6898	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6899	} while (0)
6900
6901	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6902	do { \
6903	AssertPtr(pVCpu); \
6904	AssertPtr(pMixedCtx); \
6905	AssertPtr(pVmxTransient); \
6906	Assert(pVmxTransient->fVMEntryFailed == false); \
6907	Assert(ASMIntAreEnabled()); \
6908	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6909	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6910	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6911	(unsigned)pVCpu->idCpu)); \
6912	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6913	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6914	VMX_ASSERT_PREEMPT_CPUID(); \
6915	} while (0)
6916	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6917	do { \
6918	LogFunc(("\n")); \
6919	} while(0)
6920	#else /* Release builds */
6921	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6922	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6923	#endif
6924
6925
6926	/**
6927	* Advances the guest RIP after reading it from the VMCS.
6928	*
6929	* @returns VBox status code.
6930	* @param pVCpu Pointer to the VMCPU.
6931	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6932	* out-of-sync. Make sure to update the required fields
6933	* before using them.
6934	* @param pVmxTransient Pointer to the VMX transient structure.
6935	*
6936	* @remarks No-long-jump zone!!!
6937	*/
6938	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6939	{
6940	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6941	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6942	AssertRCReturn(rc, rc);
6943
6944	pMixedCtx->rip += pVmxTransient->cbInstr;
6945	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6946	return rc;
6947	}
6948
6949
6950	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6951	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6952	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6953	/**
6954	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6955	*/
6956	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6957	{
6958	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6959	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6960	return VINF_SUCCESS;
6961	}
6962
6963
6964	/**
6965	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6966	*/
6967	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6968	{
6969	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6970	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6971
6972	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6973	AssertRCReturn(rc, rc);
6974
6975	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6976	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6977	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6978
6979	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6980	{
6981	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6982	return VINF_EM_RAW_INTERRUPT;
6983	}
6984
6985	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6986	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6987	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6988	{
6989	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6990	return VINF_SUCCESS;
6991	}
6992	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6993	{
6994	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6995	return rc;
6996	}
6997
6998	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6999	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7000	switch (uIntrType)
7001	{
7002	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7003	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7004	/* no break */
7005	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
7006	{
7007	switch (uVector)
7008	{
7009	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7010	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7011	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7012	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7013	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7014	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7015	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7016	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7017	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7018	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7019	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7020	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7021	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7022	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7023	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7024	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7025	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7026	#endif
7027	default:
7028	{
7029	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7030	AssertRCReturn(rc, rc);
7031
7032	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7033	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7034	{
7035	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7036	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7037	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7038	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7039	AssertRCReturn(rc, rc);
7040	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7041	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7042	0 /* GCPtrFaultAddress */);
7043	AssertRCReturn(rc, rc);
7044	}
7045	else
7046	{
7047	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7048	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7049	}
7050	break;
7051	}
7052	}
7053	break;
7054	}
7055
7056	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7057	default:
7058	{
7059	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7060	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7061	break;
7062	}
7063	}
7064	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7065	return rc;
7066	}
7067
7068
7069	/**
7070	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7071	*/
7072	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7073	{
7074	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7075
7076	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7077	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7078	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7079	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7080	AssertRCReturn(rc, rc);
7081
7082	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7083	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7084	return VINF_SUCCESS;
7085	}
7086
7087
7088	/**
7089	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7090	*/
7091	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7092	{
7093	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7094	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7095	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7096	}
7097
7098
7099	/**
7100	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7101	*/
7102	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7103	{
7104	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7106	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7107	}
7108
7109
7110	/**
7111	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7112	*/
7113	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7114	{
7115	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7117	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7118	}
7119
7120
7121	/**
7122	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7123	*/
7124	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7125	{
7126	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7127	PVM pVM = pVCpu->CTX_SUFF(pVM);
7128	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7129	if (RT_LIKELY(rc == VINF_SUCCESS))
7130	{
7131	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7132	Assert(pVmxTransient->cbInstr == 2);
7133	}
7134	else
7135	{
7136	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7137	rc = VERR_EM_INTERPRETER;
7138	}
7139	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7140	return rc;
7141	}
7142
7143
7144	/**
7145	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7146	*/
7147	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7148	{
7149	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7150	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7151	AssertRCReturn(rc, rc);
7152
7153	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7154	return VINF_EM_RAW_EMULATE_INSTR;
7155
7156	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7157	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7158	}
7159
7160
7161	/**
7162	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7163	*/
7164	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7165	{
7166	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7167	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7168	AssertRCReturn(rc, rc);
7169
7170	PVM pVM = pVCpu->CTX_SUFF(pVM);
7171	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7172	if (RT_LIKELY(rc == VINF_SUCCESS))
7173	{
7174	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7175	Assert(pVmxTransient->cbInstr == 2);
7176	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7177	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7178	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7179	}
7180	else
7181	{
7182	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7183	rc = VERR_EM_INTERPRETER;
7184	}
7185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7186	return rc;
7187	}
7188
7189
7190	/**
7191	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7192	*/
7193	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7194	{
7195	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7196	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7197	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7198	AssertRCReturn(rc, rc);
7199
7200	PVM pVM = pVCpu->CTX_SUFF(pVM);
7201	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7202	if (RT_LIKELY(rc == VINF_SUCCESS))
7203	{
7204	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7205	Assert(pVmxTransient->cbInstr == 3);
7206	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7207	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7208	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7209	}
7210	else
7211	{
7212	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7213	rc = VERR_EM_INTERPRETER;
7214	}
7215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7216	return rc;
7217	}
7218
7219
7220	/**
7221	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7222	*/
7223	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7224	{
7225	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7226	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7227	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7228	AssertRCReturn(rc, rc);
7229
7230	PVM pVM = pVCpu->CTX_SUFF(pVM);
7231	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7232	if (RT_LIKELY(rc == VINF_SUCCESS))
7233	{
7234	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7235	Assert(pVmxTransient->cbInstr == 2);
7236	}
7237	else
7238	{
7239	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7240	rc = VERR_EM_INTERPRETER;
7241	}
7242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7243	return rc;
7244	}
7245
7246
7247	/**
7248	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7249	*/
7250	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7251	{
7252	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7253	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7254	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7255	AssertRCReturn(rc, rc);
7256
7257	PVM pVM = pVCpu->CTX_SUFF(pVM);
7258	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7259	rc = VBOXSTRICTRC_VAL(rc2);
7260	if (RT_LIKELY(rc == VINF_SUCCESS))
7261	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7262	else
7263	{
7264	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7265	pVmxTransient->uExitQualification, rc));
7266	rc = VERR_EM_INTERPRETER;
7267	}
7268	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7269	return rc;
7270	}
7271
7272
7273	/**
7274	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7275	*/
7276	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7277	{
7278	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7279	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7280	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7281	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7282	AssertRCReturn(rc, rc);
7283
7284	PVM pVM = pVCpu->CTX_SUFF(pVM);
7285	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7286	if (RT_LIKELY(rc == VINF_SUCCESS))
7287	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7288	else
7289	{
7290	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7291	rc = VERR_EM_INTERPRETER;
7292	}
7293	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7294	return rc;
7295	}
7296
7297
7298	/**
7299	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7300	*/
7301	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7302	{
7303	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7304	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7305	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7306	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7307	AssertRCReturn(rc, rc);
7308
7309	PVM pVM = pVCpu->CTX_SUFF(pVM);
7310	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7311	rc = VBOXSTRICTRC_VAL(rc2);
7312	if (RT_LIKELY( rc == VINF_SUCCESS
7313	\|\| rc == VINF_EM_HALT))
7314	{
7315	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7316	AssertRCReturn(rc3, rc3);
7317
7318	if ( rc == VINF_EM_HALT
7319	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7320	{
7321	rc = VINF_SUCCESS;
7322	}
7323	}
7324	else
7325	{
7326	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7327	rc = VERR_EM_INTERPRETER;
7328	}
7329	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7330	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7331	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7332	return rc;
7333	}
7334
7335
7336	/**
7337	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7338	*/
7339	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7340	{
7341	/*
7342	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7343	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7344	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7345	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7346	*/
7347	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7348	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7349	}
7350
7351
7352	/**
7353	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7354	*/
7355	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7356	{
7357	/*
7358	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7359	* root operation. If we get there there is something funny going on.
7360	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7361	*/
7362	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7363	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7364	}
7365
7366
7367	/**
7368	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7369	*/
7370	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7371	{
7372	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7373	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7374	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7375	}
7376
7377
7378	/**
7379	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7380	*/
7381	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7382	{
7383	/*
7384	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7385	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7386	* See Intel spec. 25.3 "Other Causes of VM-exits".
7387	*/
7388	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7389	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7390	}
7391
7392
7393	/**
7394	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7395	* VM-exit.
7396	*/
7397	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7398	{
7399	/*
7400	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7401	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7402	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7403	*/
7404	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7405	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7406	}
7407
7408
7409	/**
7410	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7411	* VM-exit.
7412	*/
7413	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7414	{
7415	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7416	return VINF_EM_RESET;
7417	}
7418
7419
7420	/**
7421	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7422	*/
7423	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7424	{
7425	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7426	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7427	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7428	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7429	AssertRCReturn(rc, rc);
7430
7431	pMixedCtx->rip++;
7432	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7433	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7434	rc = VINF_SUCCESS;
7435	else
7436	rc = VINF_EM_HALT;
7437
7438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7439	return rc;
7440	}
7441
7442
7443	/**
7444	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7445	*/
7446	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7447	{
7448	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7449	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7450	return VINF_SUCCESS;
7451	}
7452
7453
7454	/**
7455	* VM-exit handler for expiry of the VMX preemption timer.
7456	*/
7457	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7458	{
7459	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7460
7461	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7462	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7463	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7464
7465	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7466	PVM pVM = pVCpu->CTX_SUFF(pVM);
7467	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7468	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7469	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7470	}
7471
7472
7473	/**
7474	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7475	*/
7476	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7477	{
7478	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7479	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7480	/** @todo check if XSETBV is supported by the recompiler. */
7481	return VERR_EM_INTERPRETER;
7482	}
7483
7484
7485	/**
7486	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7487	*/
7488	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7489	{
7490	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7491	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7492	/** @todo implement EMInterpretInvpcid() */
7493	return VERR_EM_INTERPRETER;
7494	}
7495
7496
7497	/**
7498	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7499	* Error VM-exit.
7500	*/
7501	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7502	{
7503	uint32_t uIntrState;
7504	RTHCUINTREG uHCReg;
7505	uint64_t u64Val;
7506
7507	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7508	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7509	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7510	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7511	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7512	AssertRCReturn(rc, rc);
7513
7514	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7515	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7516	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7517	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7518
7519	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7520	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7521	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7522	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7523	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7524	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7525	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7526	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7527	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7528	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7529	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7530	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7531
7532	PVM pVM = pVCpu->CTX_SUFF(pVM);
7533	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7534
7535	return VERR_VMX_INVALID_GUEST_STATE;
7536	}
7537
7538
7539	/**
7540	* VM-exit handler for VM-entry failure due to an MSR-load
7541	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7542	*/
7543	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7544	{
7545	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7546	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7547	}
7548
7549
7550	/**
7551	* VM-exit handler for VM-entry failure due to a machine-check event
7552	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7553	*/
7554	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7555	{
7556	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7557	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7558	}
7559
7560
7561	/**
7562	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7563	* theory.
7564	*/
7565	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7566	{
7567	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7568	return VERR_VMX_UNDEFINED_EXIT_CODE;
7569	}
7570
7571
7572	/**
7573	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7574	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7575	* Conditional VM-exit.
7576	*/
7577	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7578	{
7579	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7580	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7581	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7582	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7583	return VERR_EM_INTERPRETER;
7584	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7585	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7586	}
7587
7588
7589	/**
7590	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7591	*/
7592	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7593	{
7594	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7595	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7597	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7598	return VERR_EM_INTERPRETER;
7599	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7600	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7601	}
7602
7603
7604	/**
7605	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7606	*/
7607	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7608	{
7609	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7610	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7611	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7612	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7613	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7614	AssertRCReturn(rc, rc);
7615
7616	PVM pVM = pVCpu->CTX_SUFF(pVM);
7617	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7618	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7619	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7621
7622	/* Update RIP and continue guest execution. */
7623	if (RT_LIKELY(rc == VINF_SUCCESS))
7624	{
7625	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7626	Assert(pVmxTransient->cbInstr == 2);
7627	}
7628	return rc;
7629	}
7630
7631
7632	/**
7633	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7634	*/
7635	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7636	{
7637	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7638	PVM pVM = pVCpu->CTX_SUFF(pVM);
7639	int rc = VINF_SUCCESS;
7640
7641	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7642	if ( pVM->hm.s.fTPRPatchingActive
7643	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7644	{
7645	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7646	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7647	{
7648	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7649	AssertRC(rc);
7650	}
7651
7652	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7653	Assert(pVmxTransient->cbInstr == 2);
7654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7655	return VINF_SUCCESS;
7656	}
7657
7658	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7659	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7660	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7661	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7662	AssertRCReturn(rc, rc);
7663
7664	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7665	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7667
7668	/* Update guest-state and continue execution. */
7669	if (RT_LIKELY(rc == VINF_SUCCESS))
7670	{
7671	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7672
7673	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7674	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7675	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7676	{
7677	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7678	}
7679	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7680	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7681
7682	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7683	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7684	{
7685	switch (pMixedCtx->ecx)
7686	{
7687	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7688	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7689	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7690	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7691	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7692	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7693	}
7694	}
7695	#ifdef VBOX_STRICT
7696	else
7697	{
7698	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7699	switch (pMixedCtx->ecx)
7700	{
7701	case MSR_IA32_SYSENTER_CS:
7702	case MSR_IA32_SYSENTER_EIP:
7703	case MSR_IA32_SYSENTER_ESP:
7704	case MSR_K8_FS_BASE:
7705	case MSR_K8_GS_BASE:
7706	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7707	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7708
7709	case MSR_K8_LSTAR:
7710	case MSR_K6_STAR:
7711	case MSR_K8_SF_MASK:
7712	case MSR_K8_TSC_AUX:
7713	{
7714	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7715	pMixedCtx->ecx));
7716	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7717	}
7718
7719	case MSR_K8_KERNEL_GS_BASE:
7720	{
7721	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7722	pMixedCtx->ecx));
7723	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7724	}
7725	}
7726	}
7727	#endif /* VBOX_STRICT */
7728	}
7729	return rc;
7730	}
7731
7732
7733	/**
7734	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7735	*/
7736	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7737	{
7738	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7739	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7741	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7742	return VERR_EM_INTERPRETER;
7743	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7744	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7745	}
7746
7747
7748	/**
7749	* VM-exit handler for when the TPR value is lowered below the specified
7750	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7751	*/
7752	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7753	{
7754	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7755	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7756
7757	/*
7758	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7759	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7760	* resume guest execution.
7761	*/
7762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7763	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7764	return VINF_SUCCESS;
7765	}
7766
7767
7768	/**
7769	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7770	* VM-exit.
7771	*
7772	* @retval VINF_SUCCESS when guest execution can continue.
7773	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7774	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7775	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7776	* recompiler.
7777	*/
7778	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7779	{
7780	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7781	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7782	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7783	AssertRCReturn(rc, rc);
7784
7785	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7786	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7787	PVM pVM = pVCpu->CTX_SUFF(pVM);
7788	switch (uAccessType)
7789	{
7790	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7791	{
7792	#if 0
7793	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7794	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7795	#else
7796	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7797	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7798	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7799	#endif
7800	AssertRCReturn(rc, rc);
7801
7802	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7803	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7804	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7805	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7806
7807	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7808	{
7809	case 0: /* CR0 */
7810	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7811	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7812	break;
7813	case 2: /* C2 **/
7814	/* Nothing to do here, CR2 it's not part of the VMCS. */
7815	break;
7816	case 3: /* CR3 */
7817	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7818	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7819	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7820	break;
7821	case 4: /* CR4 */
7822	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7823	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7824	break;
7825	case 8: /* CR8 */
7826	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7827	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7828	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7829	break;
7830	default:
7831	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7832	break;
7833	}
7834
7835	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7836	break;
7837	}
7838
7839	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7840	{
7841	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7842	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7843	AssertRCReturn(rc, rc);
7844	Assert( !pVM->hm.s.fNestedPaging
7845	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7846	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7847
7848	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7849	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7850	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7851
7852	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7853	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7854	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7855	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7857	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7858	break;
7859	}
7860
7861	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7862	{
7863	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7864	AssertRCReturn(rc, rc);
7865	rc = EMInterpretCLTS(pVM, pVCpu);
7866	AssertRCReturn(rc, rc);
7867	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7869	Log(("CRX CLTS write rc=%d\n", rc));
7870	break;
7871	}
7872
7873	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7874	{
7875	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7876	AssertRCReturn(rc, rc);
7877	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7878	if (RT_LIKELY(rc == VINF_SUCCESS))
7879	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7881	Log(("CRX LMSW write rc=%d\n", rc));
7882	break;
7883	}
7884
7885	default:
7886	{
7887	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7888	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7889	}
7890	}
7891
7892	/* Validate possible error codes. */
7893	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7894	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7895	if (RT_SUCCESS(rc))
7896	{
7897	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7898	AssertRCReturn(rc2, rc2);
7899	}
7900
7901	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7902	return rc;
7903	}
7904
7905
7906	/**
7907	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7908	* VM-exit.
7909	*/
7910	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7911	{
7912	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7914
7915	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7916	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7917	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7918	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7919	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7920	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7921	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7922	AssertRCReturn(rc, rc);
7923
7924	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7925	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7926	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7927	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7928	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7929	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7930	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7931
7932	/* I/O operation lookup arrays. */
7933	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7934	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7935
7936	const uint32_t cbSize = s_aIOSize[uIOWidth];
7937	const uint32_t cbInstr = pVmxTransient->cbInstr;
7938	PVM pVM = pVCpu->CTX_SUFF(pVM);
7939	if (fIOString)
7940	{
7941	/* INS/OUTS - I/O String instruction. */
7942	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7943	/** @todo for now manually disassemble later optimize by getting the fields from
7944	* the VMCS. */
7945	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7946	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7947	* segment prefix info. */
7948	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7949	if (RT_SUCCESS(rc))
7950	{
7951	if (fIOWrite)
7952	{
7953	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7954	(DISCPUMODE)pDis->uAddrMode, cbSize);
7955	rc = VBOXSTRICTRC_VAL(rc2);
7956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7957	}
7958	else
7959	{
7960	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7961	(DISCPUMODE)pDis->uAddrMode, cbSize);
7962	rc = VBOXSTRICTRC_VAL(rc2);
7963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7964	}
7965	}
7966	else
7967	{
7968	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7969	rc = VINF_EM_RAW_EMULATE_INSTR;
7970	}
7971	}
7972	else
7973	{
7974	/* IN/OUT - I/O instruction. */
7975	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7976	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7977	if (fIOWrite)
7978	{
7979	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7980	rc = VBOXSTRICTRC_VAL(rc2);
7981	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7982	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7984	}
7985	else
7986	{
7987	uint32_t u32Result = 0;
7988	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7989	rc = VBOXSTRICTRC_VAL(rc2);
7990	if (IOM_SUCCESS(rc))
7991	{
7992	/* Save result of I/O IN instr. in AL/AX/EAX. */
7993	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7994	}
7995	else if (rc == VINF_IOM_R3_IOPORT_READ)
7996	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7997	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7998	}
7999	}
8000
8001	if (IOM_SUCCESS(rc))
8002	{
8003	pMixedCtx->rip += cbInstr;
8004	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8005	if (RT_LIKELY(rc == VINF_SUCCESS))
8006	{
8007	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8008	AssertRCReturn(rc, rc);
8009
8010	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8011	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8012	{
8013	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8014	for (unsigned i = 0; i < 4; i++)
8015	{
8016	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8017	if ( ( uIOPort >= pMixedCtx->dr[i]
8018	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8019	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8020	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8021	{
8022	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8023	uint64_t uDR6 = ASMGetDR6();
8024
8025	/* Clear all breakpoint status flags and set the one we just hit. */
8026	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8027	uDR6 \|= (uint64_t)RT_BIT(i);
8028
8029	/*
8030	* Note: AMD64 Architecture Programmer's Manual 13.1:
8031	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8032	* be cleared by software after the contents have been read.
8033	*/
8034	ASMSetDR6(uDR6);
8035
8036	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8037	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8038
8039	/* Paranoia. */
8040	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8041	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8042	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8043
8044	/* Resync DR7 */
8045	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8046	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8047
8048	/* Set #DB to be injected into the VM and continue guest execution. */
8049	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8050	break;
8051	}
8052	}
8053	}
8054	}
8055	}
8056
8057	#ifdef DEBUG
8058	if (rc == VINF_IOM_R3_IOPORT_READ)
8059	Assert(!fIOWrite);
8060	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8061	Assert(fIOWrite);
8062	else
8063	{
8064	AssertMsg( RT_FAILURE(rc)
8065	\|\| rc == VINF_SUCCESS
8066	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8067	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8068	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8069	}
8070	#endif
8071
8072	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8073	return rc;
8074	}
8075
8076
8077	/**
8078	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8079	* VM-exit.
8080	*/
8081	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8082	{
8083	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8084
8085	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8086	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8087	AssertRCReturn(rc, rc);
8088	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8089	{
8090	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8091	AssertRCReturn(rc, rc);
8092	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8093	{
8094	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8095	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8096	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8097	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8098	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8099	{
8100	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8101	Log(("Pending event on TaskSwitch uIntrType=%#x uVector=%#x\n", uIntType,
8102	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
8103	Assert(!pVCpu->hm.s.Event.fPending);
8104	pVCpu->hm.s.Event.fPending = true;
8105	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8106	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8107	AssertRCReturn(rc, rc);
8108	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
8109	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8110	else
8111	pVCpu->hm.s.Event.u32ErrCode = 0;
8112	}
8113	}
8114	}
8115	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8116	* emulation. */
8117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8118	return VERR_EM_INTERPRETER;
8119	}
8120
8121
8122	/**
8123	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8124	*/
8125	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8126	{
8127	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8128	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8129	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8130	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8131	AssertRCReturn(rc, rc);
8132	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8133	return VINF_EM_DBG_STOP;
8134	}
8135
8136
8137	/**
8138	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8139	*/
8140	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8141	{
8142	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8143	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8144
8145	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8146	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8147	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8148	return VINF_SUCCESS;
8149	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8150	return rc;
8151
8152	#if 0
8153	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8154	* just sync the whole thing. */
8155	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8156	#else
8157	/* Aggressive state sync. for now. */
8158	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8159	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8160	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8161	#endif
8162	AssertRCReturn(rc, rc);
8163
8164	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8165	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8166	switch (uAccessType)
8167	{
8168	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8169	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8170	{
8171	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8172	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8173	{
8174	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8175	}
8176
8177	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8178	GCPhys &= PAGE_BASE_GC_MASK;
8179	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8180	PVM pVM = pVCpu->CTX_SUFF(pVM);
8181	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8182	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8183	CPUMCTX2CORE(pMixedCtx), GCPhys);
8184	rc = VBOXSTRICTRC_VAL(rc2);
8185	Log(("ApicAccess %RGp %#x rc=%d\n", GCPhys,
8186	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification), rc));
8187	if ( rc == VINF_SUCCESS
8188	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8189	\|\| rc == VERR_PAGE_NOT_PRESENT)
8190	{
8191	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8192	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8193	rc = VINF_SUCCESS;
8194	}
8195	break;
8196	}
8197
8198	default:
8199	rc = VINF_EM_RAW_EMULATE_INSTR;
8200	break;
8201	}
8202
8203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8204	return rc;
8205	}
8206
8207
8208	/**
8209	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8210	* VM-exit.
8211	*/
8212	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8213	{
8214	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8215
8216	/* We should -not- get this VM-exit if the guest is debugging. */
8217	if (CPUMIsGuestDebugStateActive(pVCpu))
8218	{
8219	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8220	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8221	}
8222
8223	int rc = VERR_INTERNAL_ERROR_5;
8224	if ( !DBGFIsStepping(pVCpu)
8225	&& !CPUMIsHyperDebugStateActive(pVCpu))
8226	{
8227	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8228
8229	/* Don't intercept MOV DRx. */
8230	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8231	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8232	AssertRCReturn(rc, rc);
8233
8234	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8235	PVM pVM = pVCpu->CTX_SUFF(pVM);
8236	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8237	AssertRC(rc);
8238	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8239
8240	#ifdef VBOX_WITH_STATISTICS
8241	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8242	AssertRCReturn(rc, rc);
8243	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8245	else
8246	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8247	#endif
8248	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8249	return VINF_SUCCESS;
8250	}
8251
8252	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8253	* time and restore DRx registers afterwards */
8254	/*
8255	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8256	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8257	*/
8258	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8259	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8260	AssertRCReturn(rc, rc);
8261
8262	PVM pVM = pVCpu->CTX_SUFF(pVM);
8263	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8264	{
8265	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8266	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8267	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8268	if (RT_SUCCESS(rc))
8269	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8271	}
8272	else
8273	{
8274	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8275	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8276	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8277	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8278	}
8279
8280	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8281	if (RT_SUCCESS(rc))
8282	{
8283	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8284	AssertRCReturn(rc2, rc2);
8285	}
8286	return rc;
8287	}
8288
8289
8290	/**
8291	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8292	* Conditional VM-exit.
8293	*/
8294	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8295	{
8296	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8297	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8298
8299	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8300	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8301	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8302	return VINF_SUCCESS;
8303	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8304	return rc;
8305
8306	RTGCPHYS GCPhys = 0;
8307	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8308
8309	#if 0
8310	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8311	#else
8312	/* Aggressive state sync. for now. */
8313	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8314	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8315	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8316	#endif
8317	AssertRCReturn(rc, rc);
8318
8319	/*
8320	* If we succeed, resume guest execution.
8321	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8322	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8323	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8324	* weird case. See @bugref{6043}.
8325	*/
8326	PVM pVM = pVCpu->CTX_SUFF(pVM);
8327	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8328	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8329	rc = VBOXSTRICTRC_VAL(rc2);
8330	if ( rc == VINF_SUCCESS
8331	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8332	\|\| rc == VERR_PAGE_NOT_PRESENT)
8333	{
8334	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8335	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8336	return VINF_SUCCESS;
8337	}
8338	return rc;
8339	}
8340
8341
8342	/**
8343	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8344	* VM-exit.
8345	*/
8346	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8347	{
8348	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8349	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8350
8351	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8352	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8353	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8354	return VINF_SUCCESS;
8355	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8356	return rc;
8357
8358	RTGCPHYS GCPhys = 0;
8359	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8360	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8361	#if 0
8362	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8363	#else
8364	/* Aggressive state sync. for now. */
8365	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8366	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8367	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8368	#endif
8369	AssertRCReturn(rc, rc);
8370
8371	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8372	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8373
8374	RTGCUINT uErrorCode = 0;
8375	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8376	uErrorCode \|= X86_TRAP_PF_ID;
8377	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8378	uErrorCode \|= X86_TRAP_PF_RW;
8379	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8380	uErrorCode \|= X86_TRAP_PF_P;
8381
8382	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8383
8384	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8385	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8386
8387	/* Handle the pagefault trap for the nested shadow table. */
8388	PVM pVM = pVCpu->CTX_SUFF(pVM);
8389	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8390	TRPMResetTrap(pVCpu);
8391
8392	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8393	if ( rc == VINF_SUCCESS
8394	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8395	\|\| rc == VERR_PAGE_NOT_PRESENT)
8396	{
8397	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8398	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8399	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8400	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8401	return VINF_SUCCESS;
8402	}
8403
8404	Log(("EPT return to ring-3 rc=%d\n"));
8405	return rc;
8406	}
8407
8408
8409	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8410	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8411	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8412	/**
8413	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8414	*/
8415	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8416	{
8417	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8418	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8419
8420	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8421	AssertRCReturn(rc, rc);
8422
8423	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8424	{
8425	/* Old-style FPU error reporting needs some extra work. */
8426	/** @todo don't fall back to the recompiler, but do it manually. */
8427	return VERR_EM_INTERPRETER;
8428	}
8429	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8430	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8431	return rc;
8432	}
8433
8434
8435	/**
8436	* VM-exit exception handler for #BP (Breakpoint exception).
8437	*/
8438	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8439	{
8440	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8442
8443	/** @todo Try optimize this by not saving the entire guest state unless
8444	* really needed. */
8445	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8446	AssertRCReturn(rc, rc);
8447
8448	PVM pVM = pVCpu->CTX_SUFF(pVM);
8449	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8450	if (rc == VINF_EM_RAW_GUEST_TRAP)
8451	{
8452	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8453	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8454	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8455	AssertRCReturn(rc, rc);
8456
8457	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8458	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8459	}
8460
8461	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8462	return rc;
8463	}
8464
8465
8466	/**
8467	* VM-exit exception handler for #DB (Debug exception).
8468	*/
8469	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8470	{
8471	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8472	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8473
8474	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8475	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8476	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8477	AssertRCReturn(rc, rc);
8478
8479	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8480	uint64_t uDR6 = X86_DR6_INIT_VAL;
8481	uDR6 \|= (pVmxTransient->uExitQualification
8482	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8483	PVM pVM = pVCpu->CTX_SUFF(pVM);
8484	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8485	if (rc == VINF_EM_RAW_GUEST_TRAP)
8486	{
8487	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8488	pMixedCtx->dr[6] = uDR6;
8489
8490	if (CPUMIsGuestDebugStateActive(pVCpu))
8491	ASMSetDR6(pMixedCtx->dr[6]);
8492
8493	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8494	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8495
8496	/* Paranoia. */
8497	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8498	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8499	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8500
8501	/* Resync DR7. */
8502	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8503
8504	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8505	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8506	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8507	AssertRCReturn(rc,rc);
8508	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8509	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8510	return rc;
8511	}
8512	/* Return to ring 3 to deal with the debug exit code. */
8513	return rc;
8514	}
8515
8516
8517	/**
8518	* VM-exit exception handler for #NM (Device-not-available exception: floating
8519	* point exception).
8520	*/
8521	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8522	{
8523	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8524
8525	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8526	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8527	#endif
8528
8529	/* We require CR0 and EFER. EFER is always up-to-date. */
8530	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8531	AssertRCReturn(rc, rc);
8532
8533	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8534	PVM pVM = pVCpu->CTX_SUFF(pVM);
8535	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8536	if (rc == VINF_SUCCESS)
8537	{
8538	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8539	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8540	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8541	return VINF_SUCCESS;
8542	}
8543
8544	/* Forward #NM to the guest. */
8545	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8546	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8547	AssertRCReturn(rc, rc);
8548	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8549	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8550	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8551	return rc;
8552	}
8553
8554
8555	/**
8556	* VM-exit exception handler for #GP (General-protection exception).
8557	*
8558	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8559	*/
8560	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8561	{
8562	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8564
8565	int rc = VERR_INTERNAL_ERROR_5;
8566
8567	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8568	{
8569	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8570	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8571	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8572	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8573	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8574	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8575	AssertRCReturn(rc, rc);
8576	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8577	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8578	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8579	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8580	return rc;
8581	#else
8582	/* We don't intercept #GP. */
8583	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8584	return VERR_VMX_UNEXPECTED_EXCEPTION;
8585	#endif
8586	}
8587
8588	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8589	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8590
8591	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8592	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8593	AssertRCReturn(rc, rc);
8594
8595	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8596	uint32_t cbOp = 0;
8597	PVM pVM = pVCpu->CTX_SUFF(pVM);
8598	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8599	if (RT_SUCCESS(rc))
8600	{
8601	rc = VINF_SUCCESS;
8602	Assert(cbOp == pDis->cbInstr);
8603	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8604	switch (pDis->pCurInstr->uOpcode)
8605	{
8606	case OP_CLI:
8607	pMixedCtx->eflags.Bits.u1IF = 0;
8608	pMixedCtx->rip += pDis->cbInstr;
8609	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8611	break;
8612
8613	case OP_STI:
8614	pMixedCtx->eflags.Bits.u1IF = 1;
8615	pMixedCtx->rip += pDis->cbInstr;
8616	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8617	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8618	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8620	break;
8621
8622	case OP_HLT:
8623	rc = VINF_EM_HALT;
8624	pMixedCtx->rip += pDis->cbInstr;
8625	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8627	break;
8628
8629	case OP_POPF:
8630	{
8631	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8632	uint32_t cbParm = 0;
8633	uint32_t uMask = 0;
8634	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8635	{
8636	cbParm = 4;
8637	uMask = 0xffffffff;
8638	}
8639	else
8640	{
8641	cbParm = 2;
8642	uMask = 0xffff;
8643	}
8644
8645	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8646	RTGCPTR GCPtrStack = 0;
8647	X86EFLAGS uEflags;
8648	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8649	&GCPtrStack);
8650	if (RT_SUCCESS(rc))
8651	{
8652	Assert(sizeof(uEflags.u32) >= cbParm);
8653	uEflags.u32 = 0;
8654	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8655	}
8656	if (RT_FAILURE(rc))
8657	{
8658	rc = VERR_EM_INTERPRETER;
8659	break;
8660	}
8661	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8662	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8663	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8664	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8665	pMixedCtx->eflags.Bits.u1RF = 0;
8666	pMixedCtx->esp += cbParm;
8667	pMixedCtx->esp &= uMask;
8668	pMixedCtx->rip += pDis->cbInstr;
8669	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8670	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8671	break;
8672	}
8673
8674	case OP_PUSHF:
8675	{
8676	uint32_t cbParm = 0;
8677	uint32_t uMask = 0;
8678	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8679	{
8680	cbParm = 4;
8681	uMask = 0xffffffff;
8682	}
8683	else
8684	{
8685	cbParm = 2;
8686	uMask = 0xffff;
8687	}
8688
8689	/* Get the stack pointer & push the contents of eflags onto the stack. */
8690	RTGCPTR GCPtrStack = 0;
8691	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8692	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8693	if (RT_FAILURE(rc))
8694	{
8695	rc = VERR_EM_INTERPRETER;
8696	break;
8697	}
8698	X86EFLAGS uEflags;
8699	uEflags = pMixedCtx->eflags;
8700	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8701	uEflags.Bits.u1RF = 0;
8702	uEflags.Bits.u1VM = 0;
8703
8704	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8705	if (RT_FAILURE(rc))
8706	{
8707	rc = VERR_EM_INTERPRETER;
8708	break;
8709	}
8710	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8711	pMixedCtx->esp -= cbParm;
8712	pMixedCtx->esp &= uMask;
8713	pMixedCtx->rip += pDis->cbInstr;
8714	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8716	break;
8717	}
8718
8719	case OP_IRET:
8720	{
8721	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8722	* instruction reference. */
8723	RTGCPTR GCPtrStack = 0;
8724	uint32_t uMask = 0xffff;
8725	uint16_t aIretFrame[3];
8726	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8727	{
8728	rc = VERR_EM_INTERPRETER;
8729	break;
8730	}
8731	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8732	&GCPtrStack);
8733	if (RT_SUCCESS(rc))
8734	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8735	if (RT_FAILURE(rc))
8736	{
8737	rc = VERR_EM_INTERPRETER;
8738	break;
8739	}
8740	pMixedCtx->eip = 0;
8741	pMixedCtx->ip = aIretFrame[0];
8742	pMixedCtx->cs.Sel = aIretFrame[1];
8743	pMixedCtx->cs.ValidSel = aIretFrame[1];
8744	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8745	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8746	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8747	pMixedCtx->sp += sizeof(aIretFrame);
8748	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8749	\| HM_CHANGED_GUEST_RFLAGS;
8750	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8751	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8752	break;
8753	}
8754
8755	case OP_INT:
8756	{
8757	uint16_t uVector = pDis->Param1.uValue & 0xff;
8758	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8760	break;
8761	}
8762
8763	case OP_INTO:
8764	{
8765	if (pMixedCtx->eflags.Bits.u1OF)
8766	{
8767	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8769	}
8770	break;
8771	}
8772
8773	default:
8774	{
8775	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8776	EMCODETYPE_SUPERVISOR);
8777	rc = VBOXSTRICTRC_VAL(rc2);
8778	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8779	Log(("#GP rc=%Rrc\n", rc));
8780	break;
8781	}
8782	}
8783	}
8784	else
8785	rc = VERR_EM_INTERPRETER;
8786
8787	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8788	("#GP Unexpected rc=%Rrc\n", rc));
8789	return rc;
8790	}
8791
8792
8793	/**
8794	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8795	* the exception reported in the VMX transient structure back into the VM.
8796	*
8797	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8798	* VMX transient structure to be up-to-date.
8799	*/
8800	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8801	{
8802	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8803
8804	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8805	hmR0VmxCheckExitDueToEventDelivery(). */
8806	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8807	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8808	AssertRCReturn(rc, rc);
8809	Assert(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO);
8810	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8811	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8812	return VINF_SUCCESS;
8813	}
8814
8815
8816	/**
8817	* VM-exit exception handler for #PF (Page-fault exception).
8818	*/
8819	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8820	{
8821	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8822	PVM pVM = pVCpu->CTX_SUFF(pVM);
8823	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8824	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8825	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8826	AssertRCReturn(rc, rc);
8827
8828	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8829	if (pVM->hm.s.fNestedPaging)
8830	{
8831	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8832	{
8833	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8834	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8835	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8836	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8837	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8838	}
8839	else
8840	{
8841	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8842	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8843	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8844	Log(("Pending #DF due to vectoring #PF. NP\n"));
8845	}
8846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8847	return rc;
8848	}
8849	#else
8850	Assert(!pVM->hm.s.fNestedPaging);
8851	#endif
8852
8853	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8854	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8855	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8856	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8857	AssertRCReturn(rc, rc);
8858	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8859	if ( pVM->hm.s.fTRPPatchingAllowed
8860	&& pVM->hm.s.pGuestPatchMem
8861	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8862	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8863	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8864	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8865	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8866	{
8867	RTGCPHYS GCPhys;
8868	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8869	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8870	if ( rc == VINF_SUCCESS
8871	&& GCPhys == GCPhysApicBase)
8872	{
8873	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8874	AssertRCReturn(rc, rc);
8875
8876	/* Only attempt to patch the instruction once. */
8877	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8878	if (!pPatch)
8879	return VINF_EM_HM_PATCH_TPR_INSTR;
8880	}
8881	}
8882	#endif
8883
8884	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8885	AssertRCReturn(rc, rc);
8886
8887	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8888	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode));
8889
8890	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8891	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8892	(RTGCPTR)pVmxTransient->uExitQualification);
8893
8894	Log(("#PF: rc=%Rrc\n", rc));
8895	if (rc == VINF_SUCCESS)
8896	{
8897	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8898	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8899	* memory? We don't update the whole state here... */
8900	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8901	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8902	TRPMResetTrap(pVCpu);
8903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8904	return rc;
8905	}
8906	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8907	{
8908	if (!pVmxTransient->fVectoringPF)
8909	{
8910	/* It's a guest page fault and needs to be reflected to the guest. */
8911	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8912	TRPMResetTrap(pVCpu);
8913	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8914	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8915	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8916	0 /* cbInstr */, uGstErrorCode, pMixedCtx->cr2);
8917	}
8918	else
8919	{
8920	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8921	TRPMResetTrap(pVCpu);
8922	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8923	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8924	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8925	}
8926
8927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8928	return VINF_SUCCESS;
8929	}
8930
8931	TRPMResetTrap(pVCpu);
8932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8933	return rc;
8934	}
8935

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45576

Download in other formats: