HMVMXR0.cpp@ 45542

Last change on this file since 45542 was 45534, checked in by vboxsync, 12 years ago
VMM: use of symbolic names and asserts.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 372.6 KB

Line
1	/* $Id: HMVMXR0.cpp 45534 2013-04-13 16:16:43Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53	#define VMX_FLUSH_TAGGED_TLB_EPT_VPID 0
54	#define VMX_FLUSH_TAGGED_TLB_EPT 1
55	#define VMX_FLUSH_TAGGED_TLB_VPID 2
56	#define VMX_FLUSH_TAGGED_TLB_NONE 3
57
58	/**
59	* Updated-guest-state flags.
60	*/
61	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
62	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
63	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
64	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
65	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
66	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
67	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
68	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
69	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
70	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
71	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
72	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
73	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
74	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
75	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
76	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
77	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
78	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
79	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
80	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
81	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
82	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
83	\| VMX_UPDATED_GUEST_RIP \
84	\| VMX_UPDATED_GUEST_RSP \
85	\| VMX_UPDATED_GUEST_RFLAGS \
86	\| VMX_UPDATED_GUEST_CR0 \
87	\| VMX_UPDATED_GUEST_CR3 \
88	\| VMX_UPDATED_GUEST_CR4 \
89	\| VMX_UPDATED_GUEST_GDTR \
90	\| VMX_UPDATED_GUEST_IDTR \
91	\| VMX_UPDATED_GUEST_LDTR \
92	\| VMX_UPDATED_GUEST_TR \
93	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
94	\| VMX_UPDATED_GUEST_DEBUG \
95	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
96	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
97	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
98	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
99	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
100	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
101	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
102	\| VMX_UPDATED_GUEST_APIC_STATE)
103
104	/**
105	* Flags to skip redundant reads of some common VMCS fields.
106	*/
107	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
108	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
109	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
110	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
111	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
112	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
113
114	/**
115	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
116	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
117	* we have Nested Paging support.
118	*/
119	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/** Maximum VM-instruction error number. */
128	#define VMX_INSTR_ERROR_MAX 28
129
130	/*******************************************************************************
131	* Structures and Typedefs *
132	*******************************************************************************/
133	/**
134	* A state structure for holding miscellaneous information across
135	* VMX non-root operation and restored after the transition.
136	*/
137	typedef struct VMXTRANSIENT
138	{
139	/** The host's rflags/eflags. */
140	RTCCUINTREG uEFlags;
141	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
142	uint64_t u64LStarMsr;
143	/** The guest's TPR value used for TPR shadowing. */
144	uint8_t u8GuestTpr;
145
146	/** The basic VM-exit reason. */
147	uint16_t uExitReason;
148	/** The VM-exit exit qualification. */
149	RTGCUINTPTR uExitQualification;
150	/** The VM-exit interruption error code. */
151	uint32_t uExitIntrErrorCode;
152
153	/** The VM-exit interruption-information field. */
154	uint32_t uExitIntrInfo;
155	/** Whether the VM-entry failed or not. */
156	bool fVMEntryFailed;
157	/** The VM-exit instruction-length field. */
158	uint32_t cbInstr;
159
160	/** The VM-entry interruption-information field. */
161	uint32_t uEntryIntrInfo;
162	/** The VM-entry exception error code field. */
163	uint32_t uEntryXcptErrorCode;
164	/** The VM-entry instruction length field. */
165	uint32_t cbEntryInstr;
166
167	/** IDT-vectoring information field. */
168	uint32_t uIdtVectoringInfo;
169	/** IDT-vectoring error code. */
170	uint32_t uIdtVectoringErrorCode;
171
172	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
173	uint32_t fVmcsFieldsRead;
174	/** Whether TSC-offsetting should be setup before VM-entry. */
175	bool fUpdateTscOffsettingAndPreemptTimer;
176	/** Whether the VM-exit was caused by a page-fault during delivery of a
177	* contributary exception or a page-fault. */
178	bool fVectoringPF;
179	} VMXTRANSIENT, *PVMXTRANSIENT;
180
181	/**
182	* MSR-bitmap read permissions.
183	*/
184	typedef enum VMXMSREXITREAD
185	{
186	/** Reading this MSR causes a VM-exit. */
187	VMXMSREXIT_INTERCEPT_READ = 0xb,
188	/** Reading this MSR does not cause a VM-exit. */
189	VMXMSREXIT_PASSTHRU_READ
190	} VMXMSREXITREAD;
191
192	/**
193	* MSR-bitmap write permissions.
194	*/
195	typedef enum VMXMSREXITWRITE
196	{
197	/** Writing to this MSR causes a VM-exit. */
198	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
199	/** Writing to this MSR does not cause a VM-exit. */
200	VMXMSREXIT_PASSTHRU_WRITE
201	} VMXMSREXITWRITE;
202
203	/*******************************************************************************
204	* Internal Functions *
205	*******************************************************************************/
206	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
207	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
208	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
209	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
210	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
211	#endif
212	#if 1
213	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
214	#endif
215
216	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267
268	/*******************************************************************************
269	* Global Variables *
270	*******************************************************************************/
271	/**
272	* VM-exit handler.
273	*
274	* @returns VBox status code.
275	* @param pVCpu Pointer to the VMCPU.
276	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
277	* out-of-sync. Make sure to update the required
278	* fields before using them.
279	* @param pVmxTransient Pointer to the VMX-transient structure.
280	*/
281	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	/** Pointer to VM-exit handler. */
283	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
284
285	/**
286	* VMX_EXIT dispatch table.
287	*/
288	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
289	{
290	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
291	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
292	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
293	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
294	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
295	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
296	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
297	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
298	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
299	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
300	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
301	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
302	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
303	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
304	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
305	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
306	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
307	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
308	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
309	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
310	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
311	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
312	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
313	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
314	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
315	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
316	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
317	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
318	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
319	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
320	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
321	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
322	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
323	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
324	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
325	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
326	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
327	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
328	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
329	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
330	/* 40 UNDEFINED */ hmR0VmxExitPause,
331	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
332	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
333	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
334	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
335	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
336	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
337	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
338	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
339	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
340	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
341	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
342	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
343	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
344	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
345	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
346	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
347	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
348	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
349	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
350	};
351
352	#ifdef VBOX_STRICT
353	static const char* const g_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
354	{
355	/* 0 */ "(Not Used)",
356	/* 1 */ "VMCALL executed in VMX root operation.",
357	/* 2 */ "VMCLEAR with invalid physical address.",
358	/* 3 */ "VMCLEAR with VMXON pointer.",
359	/* 4 */ "VMLAUNCH with non-clear VMCS.",
360	/* 5 */ "VMRESUME with non-launched VMCS.",
361	/* 6 */ "VMRESUME after VMXOFF",
362	/* 7 */ "VM entry with invalid control fields.",
363	/* 8 */ "VM entry with invalid host state fields.",
364	/* 9 */ "VMPTRLD with invalid physical address.",
365	/* 10 */ "VMPTRLD with VMXON pointer.",
366	/* 11 */ "VMPTRLD with incorrect revision identifier.",
367	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
368	/* 13 */ "VMWRITE to read-only VMCS component.",
369	/* 14 */ "(Not Used)",
370	/* 15 */ "VMXON executed in VMX root operation.",
371	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
372	/* 17 */ "VM entry with non-launched executing VMCS.",
373	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
374	/* 19 */ "VMCALL with non-clear VMCS.",
375	/* 20 */ "VMCALL with invalid VM-exit control fields.",
376	/* 21 */ "(Not Used)",
377	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
378	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
379	/* 24 */ "VMCALL with invalid SMM-monitor features.",
380	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
381	/* 26 */ "VM entry with events blocked by MOV SS.",
382	/* 27 */ "(Not Used)",
383	/* 28 */ "Invalid operand to INVEPT/INVVPID."
384	};
385	#endif
386
387	/**
388	* Updates the VM's last error record. If there was a VMX instruction error,
389	* reads the error data from the VMCS and updates VCPU's last error record as
390	* well.
391	*
392	* @param pVM Pointer to the VM.
393	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
394	* VERR_VMX_UNABLE_TO_START_VM or
395	* VERR_VMX_INVALID_VMCS_FIELD).
396	* @param rc The error code.
397	*/
398	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
399	{
400	AssertPtr(pVM);
401	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
402	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
403	{
404	AssertPtrReturnVoid(pVCpu);
405	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
406	}
407	pVM->hm.s.lLastError = rc;
408	}
409
410
411	/**
412	* Reads the VM-entry interruption-information field from the VMCS into the VMX
413	* transient structure.
414	*
415	* @returns VBox status code.
416	* @param pVmxTransient Pointer to the VMX transient structure.
417	*
418	* @remarks No-long-jump zone!!!
419	*/
420	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
421	{
422	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
423	AssertRCReturn(rc, rc);
424	return VINF_SUCCESS;
425	}
426
427
428	/**
429	* Reads the VM-entry exception error code field from the VMCS into
430	* the VMX transient structure.
431	*
432	* @returns VBox status code.
433	* @param pVmxTransient Pointer to the VMX transient structure.
434	*
435	* @remarks No-long-jump zone!!!
436	*/
437	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
438	{
439	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
440	AssertRCReturn(rc, rc);
441	return VINF_SUCCESS;
442	}
443
444
445	/**
446	* Reads the VM-entry exception error code field from the VMCS into
447	* the VMX transient structure.
448	*
449	* @returns VBox status code.
450	* @param pVCpu Pointer to the VMCPU.
451	* @param pVmxTransient Pointer to the VMX transient structure.
452	*
453	* @remarks No-long-jump zone!!!
454	*/
455	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
456	{
457	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
458	AssertRCReturn(rc, rc);
459	return VINF_SUCCESS;
460	}
461
462
463	/**
464	* Reads the VM-exit interruption-information field from the VMCS into the VMX
465	* transient structure.
466	*
467	* @returns VBox status code.
468	* @param pVCpu Pointer to the VMCPU.
469	* @param pVmxTransient Pointer to the VMX transient structure.
470	*/
471	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
472	{
473	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
474	{
475	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
476	AssertRCReturn(rc, rc);
477	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
478	}
479	return VINF_SUCCESS;
480	}
481
482
483	/**
484	* Reads the VM-exit interruption error code from the VMCS into the VMX
485	* transient structure.
486	*
487	* @returns VBox status code.
488	* @param pVCpu Pointer to the VMCPU.
489	* @param pVmxTransient Pointer to the VMX transient structure.
490	*/
491	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
492	{
493	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
494	{
495	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
496	AssertRCReturn(rc, rc);
497	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
498	}
499	return VINF_SUCCESS;
500	}
501
502
503	/**
504	* Reads the VM-exit instruction length field from the VMCS into the VMX
505	* transient structure.
506	*
507	* @returns VBox status code.
508	* @param pVCpu Pointer to the VMCPU.
509	* @param pVmxTransient Pointer to the VMX transient structure.
510	*/
511	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
512	{
513	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
514	{
515	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
516	AssertRCReturn(rc, rc);
517	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
518	}
519	return VINF_SUCCESS;
520	}
521
522
523	/**
524	* Reads the exit qualification from the VMCS into the VMX transient structure.
525	*
526	* @returns VBox status code.
527	* @param pVCpu Pointer to the VMCPU.
528	* @param pVmxTransient Pointer to the VMX transient structure.
529	*/
530	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
531	{
532	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
533	{
534	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
535	AssertRCReturn(rc, rc);
536	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
537	}
538	return VINF_SUCCESS;
539	}
540
541
542	/**
543	* Reads the IDT-vectoring information field from the VMCS into the VMX
544	* transient structure.
545	*
546	* @returns VBox status code.
547	* @param pVmxTransient Pointer to the VMX transient structure.
548	*
549	* @remarks No-long-jump zone!!!
550	*/
551	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
552	{
553	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
556	AssertRCReturn(rc, rc);
557	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
558	}
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Reads the IDT-vectoring error code from the VMCS into the VMX
565	* transient structure.
566	*
567	* @returns VBox status code.
568	* @param pVmxTransient Pointer to the VMX transient structure.
569	*/
570	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
571	{
572	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
573	{
574	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
575	AssertRCReturn(rc, rc);
576	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
577	}
578	return VINF_SUCCESS;
579	}
580
581
582	/**
583	* Enters VMX root mode operation on the current CPU.
584	*
585	* @returns VBox status code.
586	* @param pVM Pointer to the VM (optional, can be NULL, after
587	* a resume).
588	* @param HCPhysCpuPage Physical address of the VMXON region.
589	* @param pvCpuPage Pointer to the VMXON region.
590	*/
591	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
592	{
593	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
594	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
595
596	if (pVM)
597	{
598	/* Write the VMCS revision dword to the VMXON region. */
599	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
600	}
601
602	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
603	RTCCUINTREG fFlags = ASMIntDisableFlags();
604
605	/* Enable the VMX bit in CR4 if necessary. */
606	RTCCUINTREG uCr4 = ASMGetCR4();
607	if (!(uCr4 & X86_CR4_VMXE))
608	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
609
610	/* Enter VMX root mode. */
611	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
612	if (RT_FAILURE(rc))
613	ASMSetCR4(uCr4);
614
615	/* Restore interrupts. */
616	ASMSetFlags(fFlags);
617	return rc;
618	}
619
620
621	/**
622	* Exits VMX root mode operation on the current CPU.
623	*
624	* @returns VBox status code.
625	*/
626	static int hmR0VmxLeaveRootMode(void)
627	{
628	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
629	RTCCUINTREG fFlags = ASMIntDisableFlags();
630	int rc = VINF_SUCCESS;
631
632	/* If we're for some reason not in VMX root mode, then don't leave it. */
633	if (ASMGetCR4() & X86_CR4_VMXE)
634	{
635	/* Exit VMX root mode and clear the VMX bit in CR4 */
636	VMXDisable();
637	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
638	}
639	else
640	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
641
642	/* Restore interrupts. */
643	ASMSetFlags(fFlags);
644	return rc;
645	}
646
647
648	/**
649	* Allocates and maps one physically contiguous page. The allocated page is
650	* zero'd out. (Used by various VT-x structures).
651	*
652	* @returns IPRT status code.
653	* @param pMemObj Pointer to the ring-0 memory object.
654	* @param ppVirt Where to store the virtual address of the
655	* allocation.
656	* @param pPhys Where to store the physical address of the
657	* allocation.
658	*/
659	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
660	{
661	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
662	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
663	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
664
665	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
666	if (RT_FAILURE(rc))
667	return rc;
668	ppVirt = RTR0MemObjAddress(pMemObj);
669	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
670	ASMMemZero32(*ppVirt, PAGE_SIZE);
671	return VINF_SUCCESS;
672	}
673
674
675	/**
676	* Frees and unmaps an allocated physical page.
677	*
678	* @param pMemObj Pointer to the ring-0 memory object.
679	* @param ppVirt Where to re-initialize the virtual address of
680	* allocation as 0.
681	* @param pHCPhys Where to re-initialize the physical address of the
682	* allocation as 0.
683	*/
684	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
685	{
686	AssertPtr(pMemObj);
687	AssertPtr(ppVirt);
688	AssertPtr(pHCPhys);
689	if (*pMemObj != NIL_RTR0MEMOBJ)
690	{
691	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
692	AssertRC(rc);
693	*pMemObj = NIL_RTR0MEMOBJ;
694	*ppVirt = 0;
695	*pHCPhys = 0;
696	}
697	}
698
699
700	/**
701	* Worker function to free VT-x related structures.
702	*
703	* @returns IPRT status code.
704	* @param pVM Pointer to the VM.
705	*/
706	static void hmR0VmxStructsFree(PVM pVM)
707	{
708	for (VMCPUID i = 0; i < pVM->cCpus; i++)
709	{
710	PVMCPU pVCpu = &pVM->aCpus[i];
711	AssertPtr(pVCpu);
712
713	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
714	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
715
716	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
717	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
718
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
720	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
721	}
722
723	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
724	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
725	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
726	#endif
727	}
728
729
730	/**
731	* Worker function to allocate VT-x related VM structures.
732	*
733	* @returns IPRT status code.
734	* @param pVM Pointer to the VM.
735	*/
736	static int hmR0VmxStructsAlloc(PVM pVM)
737	{
738	/*
739	* Initialize members up-front so we can cleanup properly on allocation failure.
740	*/
741	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
742	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
743	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
744	pVM->hm.s.vmx.HCPhys##a_Name = 0;
745
746	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
747	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
748	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
749	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
750
751	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
752	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
753	#endif
754	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
755
756	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
757	for (VMCPUID i = 0; i < pVM->cCpus; i++)
758	{
759	PVMCPU pVCpu = &pVM->aCpus[i];
760	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
761	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
762	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
763	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
764	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
765	}
766	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
767	#undef VMXLOCAL_INIT_VM_MEMOBJ
768
769	/*
770	* Allocate all the VT-x structures.
771	*/
772	int rc = VINF_SUCCESS;
773	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
774	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
775	if (RT_FAILURE(rc))
776	goto cleanup;
777	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
778	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
779	#endif
780
781	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
782	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
783	{
784	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
785	&pVM->hm.s.vmx.HCPhysApicAccess);
786	if (RT_FAILURE(rc))
787	goto cleanup;
788	}
789
790	/*
791	* Initialize per-VCPU VT-x structures.
792	*/
793	for (VMCPUID i =0; i < pVM->cCpus; i++)
794	{
795	PVMCPU pVCpu = &pVM->aCpus[i];
796	AssertPtr(pVCpu);
797
798	/* Allocate the VM control structure (VMCS). */
799	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
800	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
801	if (RT_FAILURE(rc))
802	goto cleanup;
803
804	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
805	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
806	{
807	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
808	&pVCpu->hm.s.vmx.HCPhysVirtApic);
809	if (RT_FAILURE(rc))
810	goto cleanup;
811	}
812
813	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
814	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
815	{
816	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
817	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
818	if (RT_FAILURE(rc))
819	goto cleanup;
820	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
821	}
822
823	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
824	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
825	if (RT_FAILURE(rc))
826	goto cleanup;
827
828	/* Allocate the VM-exit MSR-load page for the host MSRs. */
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
830	if (RT_FAILURE(rc))
831	goto cleanup;
832	}
833
834	return VINF_SUCCESS;
835
836	cleanup:
837	hmR0VmxStructsFree(pVM);
838	return rc;
839	}
840
841
842	/**
843	* Does global VT-x initialization (called during module initialization).
844	*
845	* @returns VBox status code.
846	*/
847	VMMR0DECL(int) VMXR0GlobalInit(void)
848	{
849	/* Setup the main VM exit handlers. */
850	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
851	#ifdef VBOX_STRICT
852	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
853	Assert(g_apfnVMExitHandlers[i]);
854	#endif
855	return VINF_SUCCESS;
856	}
857
858
859	/**
860	* Does global VT-x termination (called during module termination).
861	*/
862	VMMR0DECL(void) VMXR0GlobalTerm()
863	{
864	/* Nothing to do currently. */
865	}
866
867
868	/**
869	* Sets up and activates VT-x on the current CPU.
870	*
871	* @returns VBox status code.
872	* @param pCpu Pointer to the global CPU info struct.
873	* @param pVM Pointer to the VM (can be NULL after a host resume
874	* operation).
875	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
876	* fEnabledByHost is true).
877	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
878	* @a fEnabledByHost is true).
879	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
880	* enable VT-x/AMD-V on the host.
881	*/
882	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
883	{
884	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
885	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
886
887	if (!fEnabledByHost)
888	{
889	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
890	if (RT_FAILURE(rc))
891	return rc;
892	}
893
894	/*
895	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
896	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
897	* each time while reusing a VPID after hitting the MaxASID limit once.
898	*/
899	if ( pVM
900	&& pVM->hm.s.vmx.fVpid
901	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
902	{
903	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
904	pCpu->fFlushAsidBeforeUse = false;
905	}
906	else
907	pCpu->fFlushAsidBeforeUse = true;
908
909	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
910	++pCpu->cTlbFlushes;
911
912	return VINF_SUCCESS;
913	}
914
915
916	/**
917	* Deactivates VT-x on the current CPU.
918	*
919	* @returns VBox status code.
920	* @param pCpu Pointer to the global CPU info struct.
921	* @param pvCpuPage Pointer to the VMXON region.
922	* @param HCPhysCpuPage Physical address of the VMXON region.
923	*/
924	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
925	{
926	NOREF(pCpu);
927	NOREF(pvCpuPage);
928	NOREF(HCPhysCpuPage);
929
930	hmR0VmxLeaveRootMode();
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Sets the permission bits for the specified MSR in the MSR bitmap.
937	*
938	* @param pVCpu Pointer to the VMCPU.
939	* @param uMSR The MSR value.
940	* @param enmRead Whether reading this MSR causes a VM-exit.
941	* @param enmWrite Whether writing this MSR causes a VM-exit.
942	*/
943	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
944	{
945	int32_t iBit;
946	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
947
948	/*
949	* Layout:
950	* 0x000 - 0x3ff - Low MSR read bits
951	* 0x400 - 0x7ff - High MSR read bits
952	* 0x800 - 0xbff - Low MSR write bits
953	* 0xc00 - 0xfff - High MSR write bits
954	*/
955	if (uMsr <= 0x00001FFF)
956	{
957	/* Pentium-compatible MSRs */
958	iBit = uMsr;
959	}
960	else if ( uMsr >= 0xC0000000
961	&& uMsr <= 0xC0001FFF)
962	{
963	/* AMD Sixth Generation x86 Processor MSRs */
964	iBit = (uMsr - 0xC0000000);
965	pbMsrBitmap += 0x400;
966	}
967	else
968	{
969	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
970	return;
971	}
972
973	Assert(iBit <= 0x1fff);
974	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
975	ASMBitSet(pbMsrBitmap, iBit);
976	else
977	ASMBitClear(pbMsrBitmap, iBit);
978
979	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
980	ASMBitSet(pbMsrBitmap + 0x800, iBit);
981	else
982	ASMBitClear(pbMsrBitmap + 0x800, iBit);
983	}
984
985
986	/**
987	* Flushes the TLB using EPT.
988	*
989	* @returns VBox status code.
990	* @param pVM Pointer to the VM.
991	* @param pVCpu Pointer to the VMCPU.
992	* @param enmFlush Type of flush.
993	*/
994	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
995	{
996	AssertPtr(pVM);
997	Assert(pVM->hm.s.fNestedPaging);
998
999	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1000
1001	uint64_t descriptor[2];
1002	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1003	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1004
1005	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1006	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1007	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1008	}
1009
1010
1011	/**
1012	* Flushes the TLB using VPID.
1013	*
1014	* @returns VBox status code.
1015	* @param pVM Pointer to the VM.
1016	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1017	* enmFlush).
1018	* @param enmFlush Type of flush.
1019	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1020	* on @a enmFlush).
1021	*/
1022	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1023	{
1024	AssertPtr(pVM);
1025	Assert(pVM->hm.s.vmx.fVpid);
1026
1027	uint64_t descriptor[2];
1028	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1029	{
1030	descriptor[0] = 0;
1031	descriptor[1] = 0;
1032	}
1033	else
1034	{
1035	AssertPtr(pVCpu);
1036	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1037	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1038	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1039	descriptor[1] = GCPtr;
1040	}
1041
1042	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1043	AssertMsg(rc == VINF_SUCCESS,
1044	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1045	if ( RT_SUCCESS(rc)
1046	&& pVCpu)
1047	{
1048	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1049	}
1050	}
1051
1052
1053	/**
1054	* Invalidates a guest page by guest virtual address. Only relevant for
1055	* EPT/VPID, otherwise there is nothing really to invalidate.
1056	*
1057	* @returns VBox status code.
1058	* @param pVM Pointer to the VM.
1059	* @param pVCpu Pointer to the VMCPU.
1060	* @param GCVirt Guest virtual address of the page to invalidate.
1061	*/
1062	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1063	{
1064	AssertPtr(pVM);
1065	AssertPtr(pVCpu);
1066	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1067
1068	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1069	if (!fFlushPending)
1070	{
1071	/*
1072	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1073	* See @bugref{6043} and @bugref{6177}.
1074	*
1075	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1076	* function maybe called in a loop with individual addresses.
1077	*/
1078	if (pVM->hm.s.vmx.fVpid)
1079	{
1080	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1081	{
1082	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1084	}
1085	else
1086	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1087	}
1088	else if (pVM->hm.s.fNestedPaging)
1089	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1090	}
1091
1092	return VINF_SUCCESS;
1093	}
1094
1095
1096	/**
1097	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1098	* otherwise there is nothing really to invalidate.
1099	*
1100	* @returns VBox status code.
1101	* @param pVM Pointer to the VM.
1102	* @param pVCpu Pointer to the VMCPU.
1103	* @param GCPhys Guest physical address of the page to invalidate.
1104	*/
1105	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1106	{
1107	LogFlowFunc(("%RGp\n", GCPhys));
1108
1109	/*
1110	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1111	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1112	* This function might be called in a loop.
1113	*/
1114	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1115	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1116	return VINF_SUCCESS;
1117	}
1118
1119
1120	/**
1121	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1122	* case where neither EPT nor VPID is supported by the CPU.
1123	*
1124	* @param pVM Pointer to the VM.
1125	* @param pVCpu Pointer to the VMCPU.
1126	*
1127	* @remarks Called with interrupts disabled.
1128	*/
1129	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1130	{
1131	NOREF(pVM);
1132	AssertPtr(pVCpu);
1133	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1135
1136	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1137	AssertPtr(pCpu);
1138
1139	pVCpu->hm.s.TlbShootdown.cPages = 0;
1140	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1141	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1142	pVCpu->hm.s.fForceTLBFlush = false;
1143	return;
1144	}
1145
1146
1147	/**
1148	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1149	*
1150	* @param pVM Pointer to the VM.
1151	* @param pVCpu Pointer to the VMCPU.
1152	* @remarks All references to "ASID" in this function pertains to "VPID" in
1153	* Intel's nomenclature. The reason is, to avoid confusion in compare
1154	* statements since the host-CPU copies are named "ASID".
1155	*
1156	* @remarks Called with interrupts disabled.
1157	*/
1158	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1159	{
1160	AssertPtr(pVM);
1161	AssertPtr(pVCpu);
1162	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1163	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1164	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1165
1166	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1167	AssertPtr(pCpu);
1168
1169	/*
1170	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1171	* This can happen both for start & resume due to long jumps back to ring-3.
1172	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1173	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1174	*/
1175	bool fNewASID = false;
1176	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1177	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1178	{
1179	pVCpu->hm.s.fForceTLBFlush = true;
1180	fNewASID = true;
1181	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1182	}
1183
1184	/*
1185	* Check for explicit TLB shootdowns.
1186	*/
1187	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1188	{
1189	pVCpu->hm.s.fForceTLBFlush = true;
1190	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1191	}
1192
1193	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1194	if (pVCpu->hm.s.fForceTLBFlush)
1195	{
1196	if (fNewASID)
1197	{
1198	++pCpu->uCurrentAsid;
1199	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1200	{
1201	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1202	pCpu->cTlbFlushes++;
1203	pCpu->fFlushAsidBeforeUse = true;
1204	}
1205
1206	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1207	if (pCpu->fFlushAsidBeforeUse)
1208	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1209	}
1210	else
1211	{
1212	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1213	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1214	else
1215	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1216	}
1217
1218	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1219	pVCpu->hm.s.fForceTLBFlush = false;
1220	}
1221	else
1222	{
1223	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1224	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1225	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1226	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1227
1228	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1229	* not be executed. See hmQueueInvlPage() where it is commented
1230	* out. Support individual entry flushing someday. */
1231	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1232	{
1233	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1234
1235	/*
1236	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1237	* as supported by the CPU.
1238	*/
1239	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1240	{
1241	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1242	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1243	}
1244	else
1245	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1246	}
1247	else
1248	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1249	}
1250	pVCpu->hm.s.TlbShootdown.cPages = 0;
1251	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1252
1253	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1254	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1255	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1256	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1257	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1258	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1259
1260	/* Update VMCS with the VPID. */
1261	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1262	AssertRC(rc);
1263	}
1264
1265
1266	/**
1267	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1268	*
1269	* @returns VBox status code.
1270	* @param pVM Pointer to the VM.
1271	* @param pVCpu Pointer to the VMCPU.
1272	*
1273	* @remarks Called with interrupts disabled.
1274	*/
1275	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1276	{
1277	AssertPtr(pVM);
1278	AssertPtr(pVCpu);
1279	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1280	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1281
1282	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1283	AssertPtr(pCpu);
1284
1285	/*
1286	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1287	* This can happen both for start & resume due to long jumps back to ring-3.
1288	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1289	*/
1290	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1291	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1292	{
1293	pVCpu->hm.s.fForceTLBFlush = true;
1294	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1295	}
1296
1297	/* Check for explicit TLB shootdown flushes. */
1298	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1299	{
1300	pVCpu->hm.s.fForceTLBFlush = true;
1301	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1302	}
1303
1304	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1305	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1306
1307	if (pVCpu->hm.s.fForceTLBFlush)
1308	{
1309	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1310	pVCpu->hm.s.fForceTLBFlush = false;
1311	}
1312	else
1313	{
1314	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1315	* not be executed. See hmQueueInvlPage() where it is commented
1316	* out. Support individual entry flushing someday. */
1317	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1318	{
1319	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1320	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1321	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1322	}
1323	else
1324	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1325	}
1326
1327	pVCpu->hm.s.TlbShootdown.cPages = 0;
1328	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1329	}
1330
1331
1332	/**
1333	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1334	*
1335	* @returns VBox status code.
1336	* @param pVM Pointer to the VM.
1337	* @param pVCpu Pointer to the VMCPU.
1338	*
1339	* @remarks Called with interrupts disabled.
1340	*/
1341	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1342	{
1343	AssertPtr(pVM);
1344	AssertPtr(pVCpu);
1345	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1346	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1347
1348	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1349
1350	/*
1351	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1352	* This can happen both for start & resume due to long jumps back to ring-3.
1353	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1354	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1355	*/
1356	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1357	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1358	{
1359	pVCpu->hm.s.fForceTLBFlush = true;
1360	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1361	}
1362
1363	/* Check for explicit TLB shootdown flushes. */
1364	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1365	{
1366	/*
1367	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1368	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1369	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1370	*/
1371	pVCpu->hm.s.fForceTLBFlush = true;
1372	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1373	}
1374
1375	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1376	if (pVCpu->hm.s.fForceTLBFlush)
1377	{
1378	++pCpu->uCurrentAsid;
1379	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1380	{
1381	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1382	pCpu->fFlushAsidBeforeUse = true;
1383	pCpu->cTlbFlushes++;
1384	}
1385
1386	pVCpu->hm.s.fForceTLBFlush = false;
1387	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1388	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1389	if (pCpu->fFlushAsidBeforeUse)
1390	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1391	}
1392	else
1393	{
1394	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1395	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1396	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1397	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1398
1399	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1400	* not be executed. See hmQueueInvlPage() where it is commented
1401	* out. Support individual entry flushing someday. */
1402	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1403	{
1404	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1405	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1406	{
1407	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1408	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1409	}
1410	else
1411	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1412	}
1413	else
1414	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1415	}
1416
1417	pVCpu->hm.s.TlbShootdown.cPages = 0;
1418	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1419
1420	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1421	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1422	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1423	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1424	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1425	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1426
1427	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1428	AssertRC(rc);
1429	}
1430
1431
1432	/**
1433	* Flushes the guest TLB entry based on CPU capabilities.
1434	*
1435	* @param pVCpu Pointer to the VMCPU.
1436	*/
1437	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1438	{
1439	PVM pVM = pVCpu->CTX_SUFF(pVM);
1440	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1441	{
1442	case VMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1443	case VMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1444	case VMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1445	case VMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1446	default:
1447	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1448	break;
1449	}
1450	}
1451
1452
1453	/**
1454	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1455	* TLB entries from the host TLB before VM-entry.
1456	*
1457	* @returns VBox status code.
1458	* @param pVM Pointer to the VM.
1459	*/
1460	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1461	{
1462	/*
1463	* Determine optimal flush type for nested paging.
1464	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1465	* guest execution (see hmR3InitFinalizeR0()).
1466	*/
1467	if (pVM->hm.s.fNestedPaging)
1468	{
1469	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1470	{
1471	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1472	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1473	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1474	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1475	else
1476	{
1477	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1478	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1479	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1480	}
1481
1482	/* Make sure the write-back cacheable memory type for EPT is supported. */
1483	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1484	{
1485	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1486	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1487	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1488	}
1489	}
1490	else
1491	{
1492	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1493	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1494	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1495	}
1496	}
1497
1498	/*
1499	* Determine optimal flush type for VPID.
1500	*/
1501	if (pVM->hm.s.vmx.fVpid)
1502	{
1503	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1504	{
1505	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1506	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1507	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1508	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1509	else
1510	{
1511	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1512	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1513	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1515	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1516	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1517	pVM->hm.s.vmx.fVpid = false;
1518	}
1519	}
1520	else
1521	{
1522	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1523	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1524	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1525	pVM->hm.s.vmx.fVpid = false;
1526	}
1527	}
1528
1529	/*
1530	* Setup the handler for flushing tagged-TLBs.
1531	*/
1532	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1533	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT_VPID;
1534	else if (pVM->hm.s.fNestedPaging)
1535	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT;
1536	else if (pVM->hm.s.vmx.fVpid)
1537	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_VPID;
1538	else
1539	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_NONE;
1540	return VINF_SUCCESS;
1541	}
1542
1543
1544	/**
1545	* Sets up pin-based VM-execution controls in the VMCS.
1546	*
1547	* @returns VBox status code.
1548	* @param pVM Pointer to the VM.
1549	* @param pVCpu Pointer to the VMCPU.
1550	*/
1551	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1552	{
1553	AssertPtr(pVM);
1554	AssertPtr(pVCpu);
1555
1556	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1557	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1558
1559	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1560	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1561	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1562
1563	/* Enable the VMX preemption timer. */
1564	if (pVM->hm.s.vmx.fUsePreemptTimer)
1565	{
1566	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1567	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1568	}
1569
1570	if ((val & zap) != val)
1571	{
1572	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1573	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1574	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1575	}
1576
1577	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1578	AssertRCReturn(rc, rc);
1579
1580	/* Update VCPU with the currently set pin-based VM-execution controls. */
1581	pVCpu->hm.s.vmx.u32PinCtls = val;
1582	return rc;
1583	}
1584
1585
1586	/**
1587	* Sets up processor-based VM-execution controls in the VMCS.
1588	*
1589	* @returns VBox status code.
1590	* @param pVM Pointer to the VM.
1591	* @param pVMCPU Pointer to the VMCPU.
1592	*/
1593	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1594	{
1595	AssertPtr(pVM);
1596	AssertPtr(pVCpu);
1597
1598	int rc = VERR_INTERNAL_ERROR_5;
1599	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1600	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1601
1602	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1603	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1604	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1605	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1606	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1607	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1608	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1609
1610	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1611	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1612	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1613	{
1614	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1615	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1616	}
1617
1618	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1619	if (!pVM->hm.s.fNestedPaging)
1620	{
1621	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1622	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1623	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1624	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1625	}
1626
1627	/* Use TPR shadowing if supported by the CPU. */
1628	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1629	{
1630	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1631	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1632	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1633	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1634	AssertRCReturn(rc, rc);
1635
1636	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1637	/* CR8 writes causes a VM-exit based on TPR threshold. */
1638	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1639	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1640	}
1641	else
1642	{
1643	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1645	}
1646
1647	/* Use MSR-bitmaps if supported by the CPU. */
1648	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1649	{
1650	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1651
1652	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1653	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1654	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1655	AssertRCReturn(rc, rc);
1656
1657	/*
1658	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1659	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1660	*/
1661	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1662	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1663	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1664	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1665	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1666	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1667	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1668	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1669	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1670	}
1671
1672	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1673	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1674	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1675
1676	if ((val & zap) != val)
1677	{
1678	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1679	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1680	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1681	}
1682
1683	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1684	AssertRCReturn(rc, rc);
1685
1686	/* Update VCPU with the currently set processor-based VM-execution controls. */
1687	pVCpu->hm.s.vmx.u32ProcCtls = val;
1688
1689	/*
1690	* Secondary processor-based VM-execution controls.
1691	*/
1692	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1693	{
1694	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1695	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1696
1697	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1698	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1699
1700	if (pVM->hm.s.fNestedPaging)
1701	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1702	else
1703	{
1704	/*
1705	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1706	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1707	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1708	*/
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1710	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1711	}
1712
1713	if (pVM->hm.s.vmx.fVpid)
1714	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1715
1716	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1717	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1718
1719	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1720	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1721	* done dynamically. */
1722	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1723	{
1724	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1725	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1726	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1727	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1728	AssertRCReturn(rc, rc);
1729	}
1730
1731	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1732	{
1733	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1734	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1735	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1736	}
1737
1738	if ((val & zap) != val)
1739	{
1740	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1741	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1742	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1743	}
1744
1745	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1746	AssertRCReturn(rc, rc);
1747
1748	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1749	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1750	}
1751
1752	return VINF_SUCCESS;
1753	}
1754
1755
1756	/**
1757	* Sets up miscellaneous (everything other than Pin & Processor-based
1758	* VM-execution) control fields in the VMCS.
1759	*
1760	* @returns VBox status code.
1761	* @param pVM Pointer to the VM.
1762	* @param pVCpu Pointer to the VMCPU.
1763	*/
1764	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1765	{
1766	AssertPtr(pVM);
1767	AssertPtr(pVCpu);
1768
1769	int rc = VERR_GENERAL_FAILURE;
1770
1771	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1772	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1773
1774	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1775
1776	/*
1777	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1778	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1779	* We thus use the exception bitmap to control it rather than use both.
1780	*/
1781	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1782	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1783
1784	/** @todo Explore possibility of using IO-bitmaps. */
1785	/* All IO & IOIO instructions cause VM-exits. */
1786	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1787	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1788
1789	/* Setup MSR autoloading/autostoring. */
1790	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1791	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1792	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1793	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1794	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1795	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1796
1797	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1798	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1799	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1800	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1801
1802	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1803	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1804
1805	/* Setup debug controls */
1806	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1807	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1808	AssertRCReturn(rc, rc);
1809	return rc;
1810	}
1811
1812
1813	/**
1814	* Sets up the initial exception bitmap in the VMCS based on static conditions
1815	* (i.e. conditions that cannot ever change at runtime).
1816	*
1817	* @returns VBox status code.
1818	* @param pVM Pointer to the VM.
1819	* @param pVCpu Pointer to the VMCPU.
1820	*/
1821	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1822	{
1823	AssertPtr(pVM);
1824	AssertPtr(pVCpu);
1825
1826	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1827
1828	uint32_t u32XcptBitmap = 0;
1829
1830	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1831	if (!pVM->hm.s.fNestedPaging)
1832	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1833
1834	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1835	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1836	AssertRCReturn(rc, rc);
1837	return rc;
1838	}
1839
1840
1841	/**
1842	* Sets up the initial guest-state mask. The guest-state mask is consulted
1843	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1844	* for the nested virtualization case (as it would cause a VM-exit).
1845	*
1846	* @param pVCpu Pointer to the VMCPU.
1847	*/
1848	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1849	{
1850	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1851	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1852	return VINF_SUCCESS;
1853	}
1854
1855
1856	/**
1857	* Does per-VM VT-x initialization.
1858	*
1859	* @returns VBox status code.
1860	* @param pVM Pointer to the VM.
1861	*/
1862	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1863	{
1864	LogFlowFunc(("pVM=%p\n", pVM));
1865
1866	int rc = hmR0VmxStructsAlloc(pVM);
1867	if (RT_FAILURE(rc))
1868	{
1869	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1870	return rc;
1871	}
1872
1873	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1874	{
1875	PVMCPU pVCpu = &pVM->aCpus[i];
1876
1877	/* Current guest paging mode. */
1878	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1879	}
1880
1881	return VINF_SUCCESS;
1882	}
1883
1884
1885	/**
1886	* Does per-VM VT-x termination.
1887	*
1888	* @returns VBox status code.
1889	* @param pVM Pointer to the VM.
1890	*/
1891	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1892	{
1893	LogFlowFunc(("pVM=%p\n", pVM));
1894
1895	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1896	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1897	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1898	#endif
1899	hmR0VmxStructsFree(pVM);
1900	return VINF_SUCCESS;
1901	}
1902
1903
1904	/**
1905	* Sets up the VM for execution under VT-x.
1906	* This function is only called once per-VM during initalization.
1907	*
1908	* @returns VBox status code.
1909	* @param pVM Pointer to the VM.
1910	*/
1911	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1912	{
1913	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1915
1916	LogFlowFunc(("pVM=%p\n", pVM));
1917
1918	/*
1919	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1920	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1921	*/
1922	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1923	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1924	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1925	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1926	{
1927	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1928	return VERR_INTERNAL_ERROR;
1929	}
1930
1931	/* Initialize these always, see hmR3InitFinalizeR0().*/
1932	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1933	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1934
1935	/* Setup the tagged-TLB flush handlers. */
1936	int rc = hmR0VmxSetupTaggedTlb(pVM);
1937	if (RT_FAILURE(rc))
1938	{
1939	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1940	return rc;
1941	}
1942
1943	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1944	{
1945	PVMCPU pVCpu = &pVM->aCpus[i];
1946	AssertPtr(pVCpu);
1947	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1948
1949	/* Set revision dword at the beginning of the VMCS structure. */
1950	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1951
1952	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1953	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1954	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1955	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1956
1957	/* Load this VMCS as the current VMCS. */
1958	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1959	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1960	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1961
1962	/* Setup the pin-based VM-execution controls. */
1963	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1964	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1965	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1966
1967	/* Setup the processor-based VM-execution controls. */
1968	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1969	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1970	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1971
1972	/* Setup the rest (miscellaneous) VM-execution controls. */
1973	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1974	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1975	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1976
1977	/* Setup the initial exception bitmap. */
1978	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1979	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1980	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1981
1982	/* Setup the initial guest-state mask. */
1983	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1984	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1985	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1986
1987	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1988	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1989	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1990	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1991	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1992	#endif
1993
1994	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1995	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1996	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1997	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1998
1999	/* Update the last error record for this VCPU. */
2000	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2001	}
2002
2003	return VINF_SUCCESS;
2004	}
2005
2006
2007	/**
2008	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2009	* the VMCS.
2010	*
2011	* @returns VBox status code.
2012	* @param pVM Pointer to the VM.
2013	* @param pVCpu Pointer to the VMCPU.
2014	*/
2015	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2016	{
2017	RTCCUINTREG uReg = ASMGetCR0();
2018	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2019
2020	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2021	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2022	if (VMX_IS_64BIT_HOST_MODE())
2023	{
2024	uint64_t uReg = hmR0Get64bitCR3();
2025	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2026	}
2027	else
2028	#endif
2029	{
2030	uReg = ASMGetCR3();
2031	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2032	}
2033
2034	uReg = ASMGetCR4();
2035	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2036	AssertRCReturn(rc, rc);
2037	return rc;
2038	}
2039
2040
2041	/**
2042	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2043	* the host-state area in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @param pVM Pointer to the VM.
2047	* @param pVCpu Pointer to the VMCPU.
2048	*/
2049	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2050	{
2051	int rc = VERR_INTERNAL_ERROR_5;
2052	RTSEL uSelCS = 0;
2053	RTSEL uSelSS = 0;
2054	RTSEL uSelDS = 0;
2055	RTSEL uSelES = 0;
2056	RTSEL uSelFS = 0;
2057	RTSEL uSelGS = 0;
2058	RTSEL uSelTR = 0;
2059
2060	/*
2061	* Host Selector registers.
2062	*/
2063	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2064	if (VMX_IS_64BIT_HOST_MODE())
2065	{
2066	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2067	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2068	}
2069	else
2070	{
2071	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2072	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2073	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2074	}
2075	#else
2076	uSelCS = ASMGetCS();
2077	uSelSS = ASMGetSS();
2078	#endif
2079
2080	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2081	/** @todo Verify if we have any platform that actually run with DS or ES with
2082	* RPL != 0 in kernel space. */
2083	uSelDS = 0;
2084	uSelES = 0;
2085	uSelFS = 0;
2086	uSelGS = 0;
2087	uSelTR = ASMGetTR();
2088
2089	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2090	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2091	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2092	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2093	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2094	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2095	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2096	Assert(uSelCS != 0);
2097	Assert(uSelTR != 0);
2098
2099	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2100	#if 0
2101	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2102	Assert(uSelSS != 0);
2103	#endif
2104
2105	/* Write these host selector fields into the host-state area in the VMCS. */
2106	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2107	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2108	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2109	#if 0
2110	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2111	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2112	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2113	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2114	#endif
2115	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2116	AssertRCReturn(rc, rc);
2117
2118	/*
2119	* Host GDTR and IDTR.
2120	*/
2121	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2122	* be safe to -not- save and restore GDTR and IDTR in the assembly
2123	* code and just do it here and don't care if the limits are zapped on
2124	* VM-exit. */
2125	RTGDTR Gdtr;
2126	RT_ZERO(Gdtr);
2127	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2128	if (VMX_IS_64BIT_HOST_MODE())
2129	{
2130	X86XDTR64 Gtr64;
2131	X86XDTR64 Idtr64;
2132	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2133	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2134	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2135	Gdtr.cbGdt = Gdtr64.cb;
2136	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2137	}
2138	else
2139	#endif
2140	{
2141	RTIDTR Idtr;
2142	ASMGetGDTR(&Gdtr);
2143	ASMGetIDTR(&Idtr);
2144	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2145	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2146	}
2147	AssertRCReturn(rc, rc);
2148
2149	/*
2150	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2151	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2152	*/
2153	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2154	{
2155	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2156	return VERR_VMX_INVALID_HOST_STATE;
2157	}
2158
2159	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2160	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2161	if (VMX_IS_64BIT_HOST_MODE())
2162	{
2163	/* We need the 64-bit TR base for hybrid darwin. */
2164	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2165	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2166	}
2167	else
2168	#endif
2169	{
2170	uintptr_t uTRBase;
2171	#if HC_ARCH_BITS == 64
2172	uTRBase = X86DESC64_BASE(pDesc);
2173	#else
2174	uTRBase = X86DESC_BASE(pDesc);
2175	#endif
2176	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2177	}
2178	AssertRCReturn(rc, rc);
2179
2180	/*
2181	* Host FS base and GS base.
2182	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2183	* would take care of the bases. In 64-bit, the MSRs come into play.
2184	*/
2185	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2186	if (VMX_IS_64BIT_HOST_MODE())
2187	{
2188	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2189	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2190	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2191	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2192	AssertRCReturn(rc, rc);
2193	}
2194	#endif
2195	return rc;
2196	}
2197
2198
2199	/**
2200	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2201	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2202	* the host after every successful VM exit.
2203	*
2204	* @returns VBox status code.
2205	* @param pVM Pointer to the VM.
2206	* @param pVCpu Pointer to the VMCPU.
2207	*/
2208	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2209	{
2210	AssertPtr(pVCpu);
2211	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2212
2213	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2214	uint32_t cHostMsrs = 0;
2215	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2216
2217	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2218	{
2219	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2220	pHostMsr->u32Reserved = 0;
2221	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2222	if (CPUMIsGuestInLongMode(pVCpu))
2223	{
2224	/* Must match the EFER value in our 64 bits switcher. */
2225	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2226	}
2227	else
2228	#endif
2229	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2230	pHostMsr++; cHostMsrs++;
2231	}
2232
2233	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2234	if (VMX_IS_64BIT_HOST_MODE())
2235	{
2236	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2237	pHostMsr->u32Reserved = 0;
2238	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2239	pHostMsr++; cHostMsrs++;
2240	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2241	pHostMsr->u32Reserved = 0;
2242	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2243	pHostMsr++; cHostMsrs++;
2244	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2245	pHostMsr->u32Reserved = 0;
2246	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2247	pHostMsr++; cHostMsrs++;
2248	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2249	#if 0
2250	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2251	pMsr->u32Reserved = 0;
2252	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2253	pHostMsr++; cHostMsrs++;
2254	#endif
2255	}
2256	#endif
2257
2258	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2259	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2260	{
2261	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2262	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2263	}
2264
2265	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2266
2267	/*
2268	* Host Sysenter MSRs.
2269	*/
2270	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2271	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2272	if (VMX_IS_64BIT_HOST_MODE())
2273	{
2274	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2275	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2276	}
2277	else
2278	{
2279	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2280	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2281	}
2282	#elif HC_ARCH_BITS == 32
2283	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2284	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2285	#else
2286	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2287	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2288	#endif
2289	AssertRCReturn(rc, rc);
2290
2291	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2292	* hmR0VmxSetupExitCtls() !! */
2293	return rc;
2294	}
2295
2296
2297	/**
2298	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2299	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2300	* controls".
2301	*
2302	* @returns VBox status code.
2303	* @param pVM Pointer to the VM.
2304	* @param pVCpu Pointer to the VMCPU.
2305	* @param pCtx Pointer to the guest-CPU context.
2306	*
2307	* @remarks No-long-jump zone!!!
2308	*/
2309	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2310	{
2311	int rc = VINF_SUCCESS;
2312	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2313	{
2314	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2315	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2316
2317	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2318	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2319
2320	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2321	if (CPUMIsGuestInLongModeEx(pCtx))
2322	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2323	else
2324	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2325
2326	/*
2327	* The following should not be set (since we're not in SMM mode):
2328	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2329	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2330	*/
2331
2332	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2333	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2334	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2335
2336	if ((val & zap) != val)
2337	{
2338	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2339	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2340	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2341	}
2342
2343	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2344	AssertRCReturn(rc, rc);
2345
2346	/* Update VCPU with the currently set VM-exit controls. */
2347	pVCpu->hm.s.vmx.u32EntryCtls = val;
2348	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2349	}
2350	return rc;
2351	}
2352
2353
2354	/**
2355	* Sets up the VM-exit controls in the VMCS.
2356	*
2357	* @returns VBox status code.
2358	* @param pVM Pointer to the VM.
2359	* @param pVCpu Pointer to the VMCPU.
2360	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2361	* out-of-sync. Make sure to update the required fields
2362	* before using them.
2363	*
2364	* @remarks requires EFER.
2365	*/
2366	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2367	{
2368	int rc = VINF_SUCCESS;
2369	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2370	{
2371	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2372	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2373
2374	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2375	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2376
2377	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2378	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2379	if (VMX_IS_64BIT_HOST_MODE())
2380	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2381	else
2382	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2383	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2384	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2385	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2386	else
2387	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2388	#endif
2389
2390	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2391	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2392
2393	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2394	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2395	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2396	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2397	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2398
2399	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2400	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2401
2402	if ((val & zap) != val)
2403	{
2404	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2405	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2406	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2407	}
2408
2409	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2410	AssertRCReturn(rc, rc);
2411
2412	/* Update VCPU with the currently set VM-exit controls. */
2413	pVCpu->hm.s.vmx.u32ExitCtls = val;
2414	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2415	}
2416	return rc;
2417	}
2418
2419
2420	/**
2421	* Loads the guest APIC and related state.
2422	*
2423	* @returns VBox status code.
2424	* @param pVM Pointer to the VM.
2425	* @param pVCpu Pointer to the VMCPU.
2426	* @param pCtx Pointer to the guest-CPU context.
2427	*/
2428	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2429	{
2430	int rc = VINF_SUCCESS;
2431	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2432	{
2433	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2434	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2435	{
2436	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2437
2438	bool fPendingIntr = false;
2439	uint8_t u8GuestTpr = 0;
2440	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2441	AssertRCReturn(rc, rc);
2442
2443	/*
2444	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2445	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2446	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2447	* the interrupt when we VM-exit for other reasons.
2448	*/
2449	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2450	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2451	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2452	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2453
2454	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2455	AssertRCReturn(rc, rc);
2456
2457	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2458	if (pVM->hm.s.fTPRPatchingActive)
2459	{
2460	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2461	pCtx->msrLSTAR = u8GuestTpr;
2462	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2463	{
2464	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2465	if (fPendingIntr)
2466	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2467	else
2468	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2469	}
2470	}
2471	}
2472
2473	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2474	}
2475	return rc;
2476	}
2477
2478
2479	/**
2480	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2481	*
2482	* @returns
2483	* @param pVCpu Pointer to the VMCPU.
2484	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2485	* out-of-sync. Make sure to update the required fields
2486	* before using them.
2487	*
2488	* @remarks No-long-jump zone!!!
2489	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2490	*/
2491	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2492	{
2493	/*
2494	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2495	* inhibit interrupts or clear any existing interrupt-inhibition.
2496	*/
2497	uint32_t uIntrState = 0;
2498	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2499	{
2500	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2501	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS))
2502	== (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2503	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2504	{
2505	/*
2506	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2507	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2508	*/
2509	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2510	}
2511	else if (pMixedCtx->eflags.Bits.u1IF)
2512	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2513	else
2514	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2515	}
2516	return uIntrState;
2517	}
2518
2519
2520	/**
2521	* Loads the guest's interruptibility-state into the guest-state area in the
2522	* VMCS.
2523	*
2524	* @returns VBox status code.
2525	* @param pVCpu Pointer to the VMCPU.
2526	* @param uIntrState The interruptibility-state to set.
2527	*/
2528	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2529	{
2530	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2531	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2532	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2533	AssertRCReturn(rc, rc);
2534	return rc;
2535	}
2536
2537
2538	/**
2539	* Loads the guest's RIP into the guest-state area in the VMCS.
2540	*
2541	* @returns VBox status code.
2542	* @param pVM Pointer to the VM.
2543	* @param pVCpu Pointer to the VMCPU.
2544	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2545	* out-of-sync. Make sure to update the required fields
2546	* before using them.
2547	*
2548	* @remarks No-long-jump zone!!!
2549	*/
2550	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2551	{
2552	int rc = VINF_SUCCESS;
2553	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2554	{
2555	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2556	AssertRCReturn(rc, rc);
2557	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2558	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2559	}
2560	return rc;
2561	}
2562
2563
2564	/**
2565	* Loads the guest's RSP into the guest-state area in the VMCS.
2566	*
2567	* @returns VBox status code.
2568	* @param pVM Pointer to the VM.
2569	* @param pVCpu Pointer to the VMCPU.
2570	* @param pCtx Pointer to the guest-CPU context.
2571	*
2572	* @remarks No-long-jump zone!!!
2573	*/
2574	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2575	{
2576	int rc = VINF_SUCCESS;
2577	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2578	{
2579	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2580	AssertRCReturn(rc, rc);
2581	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2582	}
2583	return rc;
2584	}
2585
2586
2587	/**
2588	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2589	*
2590	* @returns VBox status code.
2591	* @param pVM Pointer to the VM.
2592	* @param pVCpu Pointer to the VMCPU.
2593	* @param pCtx Pointer to the guest-CPU context.
2594	*
2595	* @remarks No-long-jump zone!!!
2596	*/
2597	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2598	{
2599	int rc = VINF_SUCCESS;
2600	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2601	{
2602	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2603	Let us assert it as such and use native-width VMWRITE. */
2604	X86RFLAGS uRFlags = pCtx->rflags;
2605	Assert(uRFlags.u64 >> 32 == 0);
2606	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2607	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2608
2609	/*
2610	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2611	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2612	*/
2613	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2614	{
2615	Assert(pVM->hm.s.vmx.pRealModeTSS);
2616	Assert(PDMVmmDevHeapIsEnabled(pVM));
2617	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2618	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2619	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2620	}
2621
2622	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2623	AssertRCReturn(rc, rc);
2624
2625	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2626	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2627	}
2628	return rc;
2629	}
2630
2631
2632	/**
2633	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2634	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2635	* assembly code.
2636	*
2637	* @returns VBox status code.
2638	* @param pVM Pointer to the VM.
2639	* @param pVCpu Pointer to the VMCPU.
2640	* @param pCtx Pointer to the guest-CPU context.
2641	*
2642	* @remarks No-long-jump zone!!!
2643	*/
2644	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2645	{
2646	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2647	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2648	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2649	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2650	return rc;
2651	}
2652
2653
2654	/**
2655	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2656	* in the VMCS.
2657	*
2658	* @returns VBox status code.
2659	* @param pVM Pointer to the VM.
2660	* @param pVCpu Pointer to the VMCPU.
2661	* @param pCtx Pointer to the guest-CPU context.
2662	*
2663	* @remarks No-long-jump zone!!!
2664	*/
2665	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2666	{
2667	int rc = VINF_SUCCESS;
2668
2669	/*
2670	* Guest CR0.
2671	* Guest FPU.
2672	*/
2673	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2674	{
2675	uint64_t u64GuestCR0 = pCtx->cr0;
2676
2677	/* The guest's view (read access) of its CR0 is unblemished. */
2678	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2679	AssertRCReturn(rc, rc);
2680	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2681
2682	/* Setup VT-x's view of the guest CR0. */
2683	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2684	if (pVM->hm.s.fNestedPaging)
2685	{
2686	if (CPUMIsGuestPagingEnabledEx(pCtx))
2687	{
2688	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2689	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2690	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2691	}
2692	else
2693	{
2694	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2695	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2696	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2697	}
2698
2699	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2700	AssertRCReturn(rc, rc);
2701	}
2702	else
2703	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2704
2705	/*
2706	* Guest FPU bits.
2707	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2708	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2709	*/
2710	u64GuestCR0 \|= X86_CR0_NE;
2711	bool fInterceptNM = false;
2712	if (CPUMIsGuestFPUStateActive(pVCpu))
2713	{
2714	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2715	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2716	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2717	}
2718	else
2719	{
2720	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2721	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2722	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2723	}
2724
2725	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2726	bool fInterceptMF = false;
2727	if (!(pCtx->cr0 & X86_CR0_NE))
2728	fInterceptMF = true;
2729
2730	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2731	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2732	{
2733	Assert(PDMVmmDevHeapIsEnabled(pVM));
2734	Assert(pVM->hm.s.vmx.pRealModeTSS);
2735	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2736	fInterceptNM = true;
2737	fInterceptMF = true;
2738	}
2739	else
2740	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2741
2742	if (fInterceptNM)
2743	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2744	else
2745	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2746
2747	if (fInterceptMF)
2748	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2749	else
2750	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2751
2752	/* Additional intercepts for debugging, define these yourself explicitly. */
2753	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2754	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2755	\| RT_BIT(X86_XCPT_DB)
2756	\| RT_BIT(X86_XCPT_DE)
2757	\| RT_BIT(X86_XCPT_NM)
2758	\| RT_BIT(X86_XCPT_UD)
2759	\| RT_BIT(X86_XCPT_NP)
2760	\| RT_BIT(X86_XCPT_SS)
2761	\| RT_BIT(X86_XCPT_GP)
2762	\| RT_BIT(X86_XCPT_PF)
2763	\| RT_BIT(X86_XCPT_MF);
2764	#elif defined(VBOX_ALWAYS_TRAP_PF)
2765	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2766	#endif
2767
2768	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2769
2770	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2771	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2772	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2773	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2774	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2775	else
2776	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2777
2778	u64GuestCR0 \|= uSetCR0;
2779	u64GuestCR0 &= uZapCR0;
2780	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2781
2782	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2783	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2784	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2785	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2786
2787	/*
2788	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2789	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2790	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2791	*/
2792	uint64_t u64CR0Mask = 0;
2793	u64CR0Mask = X86_CR0_PE
2794	\| X86_CR0_WP
2795	\| X86_CR0_PG
2796	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2797	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2798	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2799
2800	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2801	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2802	u64CR0Mask &= ~X86_CR0_PE;
2803
2804	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2805	if (fInterceptNM)
2806	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2807	else
2808	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2809
2810	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2811	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2812	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2813	AssertRCReturn(rc, rc);
2814
2815	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2816	}
2817
2818	/*
2819	* Guest CR2.
2820	* It's always loaded in the assembler code. Nothing to do here.
2821	*/
2822
2823	/*
2824	* Guest CR3.
2825	*/
2826	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2827	{
2828	uint64_t u64GuestCR3 = 0;
2829	if (pVM->hm.s.fNestedPaging)
2830	{
2831	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2832
2833	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2834	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2835	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2836	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2837
2838	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2839	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2840	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2841
2842	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2843	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2844	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2845	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2846
2847	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2848	AssertRCReturn(rc, rc);
2849	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2850
2851	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2852	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2853	{
2854	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2855	if (CPUMIsGuestInPAEModeEx(pCtx))
2856	{
2857	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2858	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2859	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2860	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2861	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2862	AssertRCReturn(rc, rc);
2863	}
2864
2865	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2866	have Unrestricted Execution to handle the guest when it's not using paging. */
2867	u64GuestCR3 = pCtx->cr3;
2868	}
2869	else
2870	{
2871	/*
2872	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2873	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2874	* EPT takes care of translating it to host-physical addresses.
2875	*/
2876	RTGCPHYS GCPhys;
2877	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2878	Assert(PDMVmmDevHeapIsEnabled(pVM));
2879
2880	/* We obtain it here every time as the guest could have relocated this PCI region. */
2881	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2882	AssertRCReturn(rc, rc);
2883
2884	u64GuestCR3 = GCPhys;
2885	}
2886	}
2887	else
2888	{
2889	/* Non-nested paging case, just use the hypervisor's CR3. */
2890	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2891	}
2892
2893	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2894	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2895	AssertRCReturn(rc, rc);
2896
2897	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2898	}
2899
2900	/*
2901	* Guest CR4.
2902	*/
2903	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2904	{
2905	uint64_t u64GuestCR4 = pCtx->cr4;
2906
2907	/* The guest's view of its CR4 is unblemished. */
2908	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2909	AssertRCReturn(rc, rc);
2910	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2911
2912	/* Setup VT-x's view of the guest CR4. */
2913	/*
2914	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2915	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2916	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2917	*/
2918	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2919	{
2920	Assert(pVM->hm.s.vmx.pRealModeTSS);
2921	Assert(PDMVmmDevHeapIsEnabled(pVM));
2922	u64GuestCR4 &= ~X86_CR4_VME;
2923	}
2924
2925	if (pVM->hm.s.fNestedPaging)
2926	{
2927	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2928	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2929	{
2930	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2931	u64GuestCR4 \|= X86_CR4_PSE;
2932	/* Our identity mapping is a 32 bits page directory. */
2933	u64GuestCR4 &= ~X86_CR4_PAE;
2934	}
2935	/* else use guest CR4.*/
2936	}
2937	else
2938	{
2939	/*
2940	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2941	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2942	*/
2943	switch (pVCpu->hm.s.enmShadowMode)
2944	{
2945	case PGMMODE_REAL: /* Real-mode. */
2946	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2947	case PGMMODE_32_BIT: /* 32-bit paging. */
2948	{
2949	u64GuestCR4 &= ~X86_CR4_PAE;
2950	break;
2951	}
2952
2953	case PGMMODE_PAE: /* PAE paging. */
2954	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2955	{
2956	u64GuestCR4 \|= X86_CR4_PAE;
2957	break;
2958	}
2959
2960	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2961	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2962	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2963	break;
2964	#endif
2965	default:
2966	AssertFailed();
2967	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2968	}
2969	}
2970
2971	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2972	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2973	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2974	u64GuestCR4 \|= uSetCR4;
2975	u64GuestCR4 &= uZapCR4;
2976
2977	/* Write VT-x's view of the guest CR4 into the VMCS. */
2978	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2979	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2980
2981	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2982	uint64_t u64CR4Mask = 0;
2983	u64CR4Mask = X86_CR4_VME
2984	\| X86_CR4_PAE
2985	\| X86_CR4_PGE
2986	\| X86_CR4_PSE
2987	\| X86_CR4_VMXE;
2988	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2989	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2990	AssertRCReturn(rc, rc);
2991
2992	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2993	}
2994	return rc;
2995	}
2996
2997
2998	/**
2999	* Loads the guest debug registers into the guest-state area in the VMCS.
3000	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3001	*
3002	* @returns VBox status code.
3003	* @param pVM Pointer to the VM.
3004	* @param pVCpu Pointer to the VMCPU.
3005	* @param pCtx Pointer to the guest-CPU context.
3006	*
3007	* @remarks No-long-jump zone!!!
3008	*/
3009	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3010	{
3011	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3012	return VINF_SUCCESS;
3013
3014	#ifdef VBOX_STRICT
3015	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3016	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3017	{
3018	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3019	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3020	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3021	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3022	}
3023	#endif
3024
3025	int rc = VERR_INTERNAL_ERROR_5;
3026	bool fInterceptDB = false;
3027	bool fInterceptMovDRx = false;
3028	if (DBGFIsStepping(pVCpu))
3029	{
3030	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
3031	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3032	{
3033	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3034	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3035	AssertRCReturn(rc, rc);
3036	Assert(fInterceptDB == false);
3037	}
3038	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3039	{
3040	Assert(fInterceptDB == false);
3041	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3042	* just continuing? */
3043	}
3044	else
3045	fInterceptDB = true;
3046	}
3047	else
3048	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3049
3050	/*
3051	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3052	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3053	* The same for the hypervisor DRx registers, priority is for the guest here.
3054	*/
3055	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3056	&& !CPUMIsGuestDebugStateActive(pVCpu))
3057	{
3058	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3059	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3060	AssertRC(rc);
3061	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3062	Assert(fInterceptMovDRx == false);
3063	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3064	}
3065	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3066	&& !CPUMIsHyperDebugStateActive(pVCpu))
3067	{
3068	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3069	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3070	AssertRC(rc);
3071	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3072	fInterceptMovDRx = true;
3073	}
3074	else
3075	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3076
3077	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3078	if (fInterceptDB)
3079	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3080	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3081	{
3082	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3083	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3084	#endif
3085	}
3086
3087	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3088	if (fInterceptMovDRx)
3089	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3090	else
3091	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3092
3093	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3094	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3095
3096	/* The guest's view of its DR7 is unblemished. */
3097	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3098
3099	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3100	return rc;
3101	}
3102
3103
3104	#ifdef VBOX_STRICT
3105	/**
3106	* Strict function to validate segment registers.
3107	*/
3108	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3109	{
3110	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3111	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3112	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3113	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3114	{
3115	/* Protected mode checks */
3116	/* CS */
3117	Assert(pCtx->cs.Attr.n.u1Present);
3118	Assert(!(pCtx->cs.Attr.u & 0xf00));
3119	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3120	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3121	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3122	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3123	\|\| (pCtx->cs.Attr.n.u1Granularity));
3124	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3125	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3126	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3127	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3128	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3129	else
3130	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3131	/* SS */
3132	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3133	{
3134	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3135	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3136	Assert(pCtx->ss.Attr.n.u1Present);
3137	Assert(!(pCtx->ss.Attr.u & 0xf00));
3138	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3139	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3140	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3141	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3142	\|\| (pCtx->ss.Attr.n.u1Granularity));
3143	}
3144	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3145	/* CR0 might not be up-to-date here always, hence disabled. */
3146	#if 0
3147	if (!pCtx->cr0 & X86_CR0_PE)
3148	Assert(!pCtx->ss.Attr.n.u2Dpl);
3149	#endif
3150	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3151	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3152	{
3153	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3154	Assert(pCtx->ds.Attr.n.u1Present);
3155	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3156	Assert(!(pCtx->ds.Attr.u & 0xf00));
3157	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3158	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3159	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3160	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3161	\|\| (pCtx->ds.Attr.n.u1Granularity));
3162	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3163	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3164	}
3165	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3166	{
3167	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3168	Assert(pCtx->es.Attr.n.u1Present);
3169	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3170	Assert(!(pCtx->es.Attr.u & 0xf00));
3171	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3172	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3173	\|\| !(pCtx->es.Attr.n.u1Granularity));
3174	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3175	\|\| (pCtx->es.Attr.n.u1Granularity));
3176	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3177	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3178	}
3179	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3180	{
3181	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3182	Assert(pCtx->fs.Attr.n.u1Present);
3183	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3184	Assert(!(pCtx->fs.Attr.u & 0xf00));
3185	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3186	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3187	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3188	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3189	\|\| (pCtx->fs.Attr.n.u1Granularity));
3190	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3191	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3192	}
3193	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3194	{
3195	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3196	Assert(pCtx->gs.Attr.n.u1Present);
3197	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3198	Assert(!(pCtx->gs.Attr.u & 0xf00));
3199	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3200	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3201	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3202	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3203	\|\| (pCtx->gs.Attr.n.u1Granularity));
3204	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3205	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3206	}
3207	/* 64-bit capable CPUs. */
3208	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3209	Assert(!(pCtx->cs.u64Base >> 32));
3210	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3211	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3212	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3213	# endif
3214	}
3215	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3216	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3217	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3218	{
3219	/* Real and v86 mode checks. */
3220	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3221	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3222	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3223	{
3224	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3225	}
3226	else
3227	{
3228	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3229	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3230	}
3231
3232	/* CS */
3233	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3234	Assert(pCtx->cs.u32Limit == 0xffff);
3235	Assert(u32CSAttr == 0xf3);
3236	/* SS */
3237	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3238	Assert(pCtx->ss.u32Limit == 0xffff);
3239	Assert(u32SSAttr == 0xf3);
3240	/* DS */
3241	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3242	Assert(pCtx->ds.u32Limit == 0xffff);
3243	Assert(u32DSAttr == 0xf3);
3244	/* ES */
3245	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3246	Assert(pCtx->es.u32Limit == 0xffff);
3247	Assert(u32ESAttr == 0xf3);
3248	/* FS */
3249	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3250	Assert(pCtx->fs.u32Limit == 0xffff);
3251	Assert(u32FSAttr == 0xf3);
3252	/* GS */
3253	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3254	Assert(pCtx->gs.u32Limit == 0xffff);
3255	Assert(u32GSAttr == 0xf3);
3256	/* 64-bit capable CPUs. */
3257	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3258	Assert(!(pCtx->cs.u64Base >> 32));
3259	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3260	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3261	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3262	# endif
3263	}
3264	}
3265	#endif /* VBOX_STRICT */
3266
3267
3268	/**
3269	* Writes a guest segment register into the guest-state area in the VMCS.
3270	*
3271	* @returns VBox status code.
3272	* @param pVM Pointer to the VM.
3273	* @param pVCpu Pointer to the VMCPU.
3274	* @param idxSel Index of the selector in the VMCS.
3275	* @param idxLimit Index of the segment limit in the VMCS.
3276	* @param idxBase Index of the segment base in the VMCS.
3277	* @param idxAccess Index of the access rights of the segment in the VMCS.
3278	* @param pSelReg Pointer to the segment selector.
3279	* @param pCtx Pointer to the guest-CPU context.
3280	*
3281	* @remarks No-long-jump zone!!!
3282	*/
3283	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3284	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3285	{
3286	int rc;
3287	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3288	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3289	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3290	AssertRCReturn(rc, rc);
3291
3292	uint32_t u32Access = pSelReg->Attr.u;
3293	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3294	{
3295	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3296	u32Access = 0xf3;
3297	Assert(pVM->hm.s.vmx.pRealModeTSS);
3298	Assert(PDMVmmDevHeapIsEnabled(pVM));
3299	}
3300	else
3301	{
3302	/*
3303	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3304	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3305	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3306	* loaded in protected-mode have their attribute as 0.
3307	*/
3308	if (!u32Access)
3309	u32Access = VMX_SEL_UNUSABLE;
3310	}
3311
3312	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3313	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3314	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3315
3316	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3317	AssertRCReturn(rc, rc);
3318	return rc;
3319	}
3320
3321
3322	/**
3323	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3324	* into the guest-state area in the VMCS.
3325	*
3326	* @returns VBox status code.
3327	* @param pVM Pointer to the VM.
3328	* @param pVCPU Pointer to the VMCPU.
3329	* @param pCtx Pointer to the guest-CPU context.
3330	*
3331	* @remarks No-long-jump zone!!!
3332	*/
3333	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3334	{
3335	int rc = VERR_INTERNAL_ERROR_5;
3336
3337	/*
3338	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3339	*/
3340	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3341	{
3342	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3343	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3344	{
3345	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3346	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3347	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3348	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3349	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3350	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3351	}
3352
3353	#ifdef VBOX_WITH_REM
3354	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3355	{
3356	Assert(pVM->hm.s.vmx.pRealModeTSS);
3357	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3358	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3359	{
3360	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3361	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3362	&& enmGuestMode >= PGMMODE_PROTECTED)
3363	{
3364	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3365	in real-mode (e.g. OpenBSD 4.0) */
3366	REMFlushTBs(pVM);
3367	Log2(("Switch to protected mode detected!\n"));
3368	}
3369	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3370	}
3371	}
3372	#endif
3373	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3374	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3375	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3376	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3377	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3378	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3379	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3380	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3381	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3382	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3383	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3384	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3385	AssertRCReturn(rc, rc);
3386
3387	#ifdef VBOX_STRICT
3388	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3389	#endif
3390	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3391	}
3392
3393	/*
3394	* Guest TR.
3395	*/
3396	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3397	{
3398	/*
3399	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3400	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3401	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3402	*/
3403	uint16_t u16Sel = 0;
3404	uint32_t u32Limit = 0;
3405	uint64_t u64Base = 0;
3406	uint32_t u32AccessRights = 0;
3407
3408	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3409	{
3410	u16Sel = pCtx->tr.Sel;
3411	u32Limit = pCtx->tr.u32Limit;
3412	u64Base = pCtx->tr.u64Base;
3413	u32AccessRights = pCtx->tr.Attr.u;
3414	}
3415	else
3416	{
3417	Assert(pVM->hm.s.vmx.pRealModeTSS);
3418	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3419
3420	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3421	RTGCPHYS GCPhys;
3422	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3423	AssertRCReturn(rc, rc);
3424
3425	X86DESCATTR DescAttr;
3426	DescAttr.u = 0;
3427	DescAttr.n.u1Present = 1;
3428	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3429
3430	u16Sel = 0;
3431	u32Limit = HM_VTX_TSS_SIZE;
3432	u64Base = GCPhys; /* in real-mode phys = virt. */
3433	u32AccessRights = DescAttr.u;
3434	}
3435
3436	/* Validate. */
3437	Assert(!(u16Sel & RT_BIT(2)));
3438	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3439	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3440	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3441	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3442	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3443	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3444	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3445	Assert( (u32Limit & 0xfff) == 0xfff
3446	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3447	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3448	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3449
3450	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3451	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3452	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3453	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3454	AssertRCReturn(rc, rc);
3455
3456	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3457	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3458	}
3459
3460	/*
3461	* Guest GDTR.
3462	*/
3463	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3464	{
3465	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3466	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3467	AssertRCReturn(rc, rc);
3468
3469	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3470	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3471	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3472	}
3473
3474	/*
3475	* Guest LDTR.
3476	*/
3477	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3478	{
3479	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3480	uint32_t u32Access = 0;
3481	if (!pCtx->ldtr.Attr.u)
3482	u32Access = VMX_SEL_UNUSABLE;
3483	else
3484	u32Access = pCtx->ldtr.Attr.u;
3485
3486	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3487	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3488	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3489	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3490	AssertRCReturn(rc, rc);
3491
3492	/* Validate. */
3493	if (!(u32Access & VMX_SEL_UNUSABLE))
3494	{
3495	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3496	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3497	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3498	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3499	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3500	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3501	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3502	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3503	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3504	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3505	}
3506
3507	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3508	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3509	}
3510
3511	/*
3512	* Guest IDTR.
3513	*/
3514	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3515	{
3516	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3517	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3518	AssertRCReturn(rc, rc);
3519
3520	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3521	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3522	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3523	}
3524
3525	/*
3526	* Guest FS & GS base MSRs.
3527	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3528	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3529	* are only available in 64-bit mode.
3530	*/
3531	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3532	* should not be necessary to do it in assembly again. */
3533	if (CPUMIsGuestInLongModeEx(pCtx))
3534	{
3535	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3536	{
3537	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3538	AssertRCReturn(rc, rc);
3539	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3540	}
3541
3542	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3543	{
3544	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3545	AssertRCReturn(rc, rc);
3546	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3547	}
3548	}
3549	else
3550	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3551
3552	return VINF_SUCCESS;
3553	}
3554
3555
3556	/**
3557	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3558	* areas. These MSRs will automatically be loaded to the host CPU on every
3559	* successful VM entry and stored from the host CPU on every successful VM exit.
3560	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3561	*
3562	* @returns VBox status code.
3563	* @param pVM Pointer to the VM.
3564	* @param pVCpu Pointer to the VMCPU.
3565	* @param pCtx Pointer to the guest-CPU context.
3566	*
3567	* @remarks No-long-jump zone!!!
3568	*/
3569	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3570	{
3571	AssertPtr(pVCpu);
3572	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3573
3574	/*
3575	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3576	*/
3577	int rc = VINF_SUCCESS;
3578	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3579	{
3580	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3581	uint32_t cGuestMsrs = 0;
3582
3583	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3584	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3585	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3586	if (fSupportsNX \|\| fSupportsLongMode)
3587	{
3588	/** @todo support save IA32_EFER, i.e.
3589	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3590	* guest EFER need not be part of the VM-entry MSR-load area. */
3591	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3592	pGuestMsr->u32Reserved = 0;
3593	pGuestMsr->u64Value = pCtx->msrEFER;
3594	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3595	if (!CPUMIsGuestInLongModeEx(pCtx))
3596	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3597	pGuestMsr++; cGuestMsrs++;
3598	if (fSupportsLongMode)
3599	{
3600	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3601	pGuestMsr->u32Reserved = 0;
3602	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3603	pGuestMsr++; cGuestMsrs++;
3604	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3605	pGuestMsr->u32Reserved = 0;
3606	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3607	pGuestMsr++; cGuestMsrs++;
3608	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3609	pGuestMsr->u32Reserved = 0;
3610	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3611	pGuestMsr++; cGuestMsrs++;
3612	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3613	#if 0
3614	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3615	pGuestMsr->u32Reserved = 0;
3616	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3617	pGuestMsr++; cGuestMsrs++;
3618	#endif
3619	}
3620	}
3621
3622	/*
3623	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3624	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3625	*/
3626	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3627	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3628	{
3629	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3630	pGuestMsr->u32Reserved = 0;
3631	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3632	AssertRCReturn(rc, rc);
3633	pGuestMsr++; cGuestMsrs++;
3634	}
3635
3636	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3637	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3638	{
3639	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3640	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3641	}
3642
3643	/* Update the VCPU's copy of the guest MSR count. */
3644	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3646	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3647	AssertRCReturn(rc, rc);
3648
3649	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3650	}
3651
3652	/*
3653	* Guest Sysenter MSRs.
3654	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3655	* VM exits on WRMSRs for these MSRs.
3656	*/
3657	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3658	{
3659	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3661	}
3662	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3663	{
3664	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3665	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3666	}
3667	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3668	{
3669	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3670	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3671	}
3672	AssertRCReturn(rc, rc);
3673
3674	return rc;
3675	}
3676
3677
3678	/**
3679	* Loads the guest activity state into the guest-state area in the VMCS.
3680	*
3681	* @returns VBox status code.
3682	* @param pVM Pointer to the VM.
3683	* @param pVCpu Pointer to the VMCPU.
3684	* @param pCtx Pointer to the guest-CPU context.
3685	*
3686	* @remarks No-long-jump zone!!!
3687	*/
3688	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3689	{
3690	/** @todo See if we can make use of other states, e.g.
3691	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3692	int rc = VINF_SUCCESS;
3693	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3694	{
3695	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3696	AssertRCReturn(rc, rc);
3697	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3698	}
3699	return rc;
3700	}
3701
3702
3703	/**
3704	* Sets up the appropriate function to run guest code.
3705	*
3706	* @returns VBox status code.
3707	* @param pVM Pointer to the VM.
3708	* @param pVCpu Pointer to the VMCPU.
3709	* @param pCtx Pointer to the guest-CPU context.
3710	*
3711	* @remarks No-long-jump zone!!!
3712	*/
3713	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3714	{
3715	if (CPUMIsGuestInLongModeEx(pCtx))
3716	{
3717	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3718	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3719	#endif
3720	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3721	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3722	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3723	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3724	#else
3725	/* 64-bit host or hybrid host. */
3726	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3727	#endif
3728	}
3729	else
3730	{
3731	/* Guest is not in long mode, use the 32-bit handler. */
3732	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3733	}
3734	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3735	return VINF_SUCCESS;
3736	}
3737
3738
3739	/**
3740	* Wrapper for running the guest code in VT-x.
3741	*
3742	* @returns VBox strict status code.
3743	* @param pVM Pointer to the VM.
3744	* @param pVCpu Pointer to the VMCPU.
3745	* @param pCtx Pointer to the guest-CPU context.
3746	*
3747	* @remarks No-long-jump zone!!!
3748	*/
3749	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3750	{
3751	/*
3752	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3753	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3754	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3755	*/
3756	#ifdef VBOX_WITH_KERNEL_USING_XMM
3757	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3758	#else
3759	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3760	#endif
3761	}
3762
3763
3764	/**
3765	* Report world-switch error and dump some useful debug info.
3766	*
3767	* @param pVM Pointer to the VM.
3768	* @param pVCpu Pointer to the VMCPU.
3769	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3770	* @param pCtx Pointer to the guest-CPU context.
3771	* @param pVmxTransient Pointer to the VMX transient structure (only
3772	* exitReason updated).
3773	*/
3774	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3775	{
3776	Assert(pVM);
3777	Assert(pVCpu);
3778	Assert(pCtx);
3779	Assert(pVmxTransient);
3780	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3781
3782	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3783	switch (rcVMRun)
3784	{
3785	case VERR_VMX_INVALID_VMXON_PTR:
3786	AssertFailed();
3787	break;
3788	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3789	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3790	{
3791	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3792	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3793	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3794	AssertRC(rc);
3795
3796	#ifdef VBOX_STRICT
3797	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3798	pVmxTransient->uExitReason));
3799	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3800	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3801	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3802	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3803	else
3804	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3805
3806	/* VMX control bits. */
3807	uint32_t u32Val;
3808	uint64_t u64Val;
3809	RTHCUINTREG uHCReg;
3810	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3811	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3812	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3813	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3814	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3815	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3816	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3817	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3818	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3819	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3820	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3821	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3822	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3823	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3824	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3826	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3827	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3828	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3829	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3830	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3831	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3832	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3833	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3834	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3835	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3836	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3837	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3838	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3839	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3840	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3841	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3842	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3843	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3844	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3845	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3846	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3847	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3848	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3849	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3850	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3851	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3852
3853	/* Guest bits. */
3854	RTGCUINTREG uGCReg;
3855	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3856	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3857	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3858	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3859	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3860	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3861	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3862	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3863
3864	/* Host bits. */
3865	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3866	Log(("Host CR0 %#RHr\n", uHCReg));
3867	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3868	Log(("Host CR3 %#RHr\n", uHCReg));
3869	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3870	Log(("Host CR4 %#RHr\n", uHCReg));
3871
3872	RTGDTR HostGdtr;
3873	PCX86DESCHC pDesc;
3874	ASMGetGDTR(&HostGdtr);
3875	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3876	Log(("Host CS %#08x\n", u32Val));
3877	if (u32Val < HostGdtr.cbGdt)
3878	{
3879	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3880	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3881	}
3882
3883	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3884	Log(("Host DS %#08x\n", u32Val));
3885	if (u32Val < HostGdtr.cbGdt)
3886	{
3887	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3888	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3889	}
3890
3891	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3892	Log(("Host ES %#08x\n", u32Val));
3893	if (u32Val < HostGdtr.cbGdt)
3894	{
3895	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3896	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3897	}
3898
3899	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3900	Log(("Host FS %#08x\n", u32Val));
3901	if (u32Val < HostGdtr.cbGdt)
3902	{
3903	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3904	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3905	}
3906
3907	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3908	Log(("Host GS %#08x\n", u32Val));
3909	if (u32Val < HostGdtr.cbGdt)
3910	{
3911	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3912	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3913	}
3914
3915	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3916	Log(("Host SS %#08x\n", u32Val));
3917	if (u32Val < HostGdtr.cbGdt)
3918	{
3919	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3920	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3921	}
3922
3923	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3924	Log(("Host TR %#08x\n", u32Val));
3925	if (u32Val < HostGdtr.cbGdt)
3926	{
3927	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3928	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3929	}
3930
3931	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3932	Log(("Host TR Base %#RHv\n", uHCReg));
3933	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3934	Log(("Host GDTR Base %#RHv\n", uHCReg));
3935	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3936	Log(("Host IDTR Base %#RHv\n", uHCReg));
3937	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3938	Log(("Host SYSENTER CS %#08x\n", u32Val));
3939	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3940	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3941	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3942	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3943	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3944	Log(("Host RSP %#RHv\n", uHCReg));
3945	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3946	Log(("Host RIP %#RHv\n", uHCReg));
3947	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3948	if (VMX_IS_64BIT_HOST_MODE())
3949	{
3950	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3951	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3952	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3953	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3954	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3955	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3956	}
3957	# endif
3958	#endif /* VBOX_STRICT */
3959	break;
3960	}
3961
3962	default:
3963	/* Impossible */
3964	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3965	break;
3966	}
3967	NOREF(pVM);
3968	}
3969
3970
3971	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3972	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3973	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3974	#endif
3975
3976	#ifdef VBOX_STRICT
3977	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3978	{
3979	switch (idxField)
3980	{
3981	case VMX_VMCS_GUEST_RIP:
3982	case VMX_VMCS_GUEST_RSP:
3983	case VMX_VMCS_GUEST_RFLAGS:
3984	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3985	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3986	case VMX_VMCS_GUEST_CR0:
3987	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3988	case VMX_VMCS_GUEST_CR4:
3989	case VMX_VMCS_GUEST_DR7:
3990	case VMX_VMCS32_GUEST_SYSENTER_CS:
3991	case VMX_VMCS_GUEST_SYSENTER_EIP:
3992	case VMX_VMCS_GUEST_SYSENTER_ESP:
3993	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3994	case VMX_VMCS_GUEST_GDTR_BASE:
3995	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3996	case VMX_VMCS_GUEST_IDTR_BASE:
3997	case VMX_VMCS16_GUEST_FIELD_CS:
3998	case VMX_VMCS32_GUEST_CS_LIMIT:
3999	case VMX_VMCS_GUEST_CS_BASE:
4000	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4001	case VMX_VMCS16_GUEST_FIELD_DS:
4002	case VMX_VMCS32_GUEST_DS_LIMIT:
4003	case VMX_VMCS_GUEST_DS_BASE:
4004	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4005	case VMX_VMCS16_GUEST_FIELD_ES:
4006	case VMX_VMCS32_GUEST_ES_LIMIT:
4007	case VMX_VMCS_GUEST_ES_BASE:
4008	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4009	case VMX_VMCS16_GUEST_FIELD_FS:
4010	case VMX_VMCS32_GUEST_FS_LIMIT:
4011	case VMX_VMCS_GUEST_FS_BASE:
4012	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4013	case VMX_VMCS16_GUEST_FIELD_GS:
4014	case VMX_VMCS32_GUEST_GS_LIMIT:
4015	case VMX_VMCS_GUEST_GS_BASE:
4016	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4017	case VMX_VMCS16_GUEST_FIELD_SS:
4018	case VMX_VMCS32_GUEST_SS_LIMIT:
4019	case VMX_VMCS_GUEST_SS_BASE:
4020	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4021	case VMX_VMCS16_GUEST_FIELD_LDTR:
4022	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4023	case VMX_VMCS_GUEST_LDTR_BASE:
4024	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4025	case VMX_VMCS16_GUEST_FIELD_TR:
4026	case VMX_VMCS32_GUEST_TR_LIMIT:
4027	case VMX_VMCS_GUEST_TR_BASE:
4028	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4029	case VMX_VMCS32_RO_EXIT_REASON:
4030	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4031	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4032	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4033	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4034	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4035	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4036	case VMX_VMCS32_RO_IDT_INFO:
4037	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4038	case VMX_VMCS_GUEST_CR3:
4039	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4040	return true;
4041	}
4042	return false;
4043	}
4044
4045	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4046	{
4047	switch (idxField)
4048	{
4049	case VMX_VMCS_GUEST_LDTR_BASE:
4050	case VMX_VMCS_GUEST_TR_BASE:
4051	case VMX_VMCS_GUEST_GDTR_BASE:
4052	case VMX_VMCS_GUEST_IDTR_BASE:
4053	case VMX_VMCS_GUEST_SYSENTER_EIP:
4054	case VMX_VMCS_GUEST_SYSENTER_ESP:
4055	case VMX_VMCS_GUEST_CR0:
4056	case VMX_VMCS_GUEST_CR4:
4057	case VMX_VMCS_GUEST_CR3:
4058	case VMX_VMCS_GUEST_DR7:
4059	case VMX_VMCS_GUEST_RIP:
4060	case VMX_VMCS_GUEST_RSP:
4061	case VMX_VMCS_GUEST_CS_BASE:
4062	case VMX_VMCS_GUEST_DS_BASE:
4063	case VMX_VMCS_GUEST_ES_BASE:
4064	case VMX_VMCS_GUEST_FS_BASE:
4065	case VMX_VMCS_GUEST_GS_BASE:
4066	case VMX_VMCS_GUEST_SS_BASE:
4067	return true;
4068	}
4069	return false;
4070	}
4071	#endif /* VBOX_STRICT */
4072
4073	/**
4074	* Executes the specified handler in 64-bit mode.
4075	*
4076	* @returns VBox status code.
4077	* @param pVM Pointer to the VM.
4078	* @param pVCpu Pointer to the VMCPU.
4079	* @param pCtx Pointer to the guest CPU context.
4080	* @param pfnHandler Pointer to the RC handler function.
4081	* @param cbParam Number of parameters.
4082	* @param paParam Array of 32-bit parameters.
4083	*/
4084	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4085	uint32_t *paParam)
4086	{
4087	int rc, rc2;
4088	PHMGLOBLCPUINFO pCpu;
4089	RTHCPHYS HCPhysCpuPage;
4090	RTHCUINTREG uOldEFlags;
4091
4092	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4093	Assert(pfnHandler);
4094	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4095	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4096
4097	#ifdef VBOX_STRICT
4098	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4099	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4100
4101	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4102	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4103	#endif
4104
4105	/* Disable interrupts. */
4106	uOldEFlags = ASMIntDisableFlags();
4107
4108	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4109	RTCPUID idHostCpu = RTMpCpuId();
4110	CPUMR0SetLApic(pVM, idHostCpu);
4111	#endif
4112
4113	pCpu = HMR0GetCurrentCpu();
4114	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4115
4116	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4117	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4118
4119	/* Leave VMX Root Mode. */
4120	VMXDisable();
4121
4122	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4123
4124	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4125	CPUMSetHyperEIP(pVCpu, pfnHandler);
4126	for (int i = (int)cbParam - 1; i >= 0; i--)
4127	CPUMPushHyper(pVCpu, paParam[i]);
4128
4129	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4130
4131	/* Call the switcher. */
4132	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4133	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4134
4135	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4136	/* Make sure the VMX instructions don't cause #UD faults. */
4137	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4138
4139	/* Re-enter VMX Root Mode */
4140	rc2 = VMXEnable(HCPhysCpuPage);
4141	if (RT_FAILURE(rc2))
4142	{
4143	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4144	ASMSetFlags(uOldEFlags);
4145	return rc2;
4146	}
4147
4148	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4149	AssertRC(rc2);
4150	Assert(!(ASMGetFlags() & X86_EFL_IF));
4151	ASMSetFlags(uOldEFlags);
4152	return rc;
4153	}
4154
4155
4156	/**
4157	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4158	* supporting 64-bit guests.
4159	*
4160	* @returns VBox status code.
4161	* @param fResume Whether to VMLAUNCH or VMRESUME.
4162	* @param pCtx Pointer to the guest-CPU context.
4163	* @param pCache Pointer to the VMCS cache.
4164	* @param pVM Pointer to the VM.
4165	* @param pVCpu Pointer to the VMCPU.
4166	*/
4167	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4168	{
4169	uint32_t aParam[6];
4170	PHMGLOBLCPUINFO pCpu = NULL;
4171	RTHCPHYS HCPhysCpuPage = 0;
4172	int rc = VERR_INTERNAL_ERROR_5;
4173
4174	pCpu = HMR0GetCurrentCpu();
4175	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4176
4177	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4178	pCache->uPos = 1;
4179	pCache->interPD = PGMGetInterPaeCR3(pVM);
4180	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4181	#endif
4182
4183	#ifdef VBOX_STRICT
4184	pCache->TestIn.HCPhysCpuPage = 0;
4185	pCache->TestIn.HCPhysVmcs = 0;
4186	pCache->TestIn.pCache = 0;
4187	pCache->TestOut.HCPhysVmcs = 0;
4188	pCache->TestOut.pCache = 0;
4189	pCache->TestOut.pCtx = 0;
4190	pCache->TestOut.eflags = 0;
4191	#endif
4192
4193	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4194	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4195	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4196	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4197	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4198	aParam[5] = 0;
4199
4200	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4201	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4202	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4203	#endif
4204	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4205
4206	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4207	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4208	Assert(pCtx->dr[4] == 10);
4209	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4210	#endif
4211
4212	#ifdef VBOX_STRICT
4213	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4214	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4215	pVCpu->hm.s.vmx.HCPhysVmcs));
4216	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4217	pCache->TestOut.HCPhysVmcs));
4218	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4219	pCache->TestOut.pCache));
4220	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4221	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4222	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4223	pCache->TestOut.pCtx));
4224	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4225	#endif
4226	return rc;
4227	}
4228
4229
4230	/**
4231	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4232	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4233	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4234	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4235	*
4236	* @returns VBox status code.
4237	* @param pVM Pointer to the VM.
4238	* @param pVCpu Pointer to the VMCPU.
4239	*/
4240	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4241	{
4242	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4243	{ \
4244	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4245	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4246	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4247	}
4248
4249	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4250	{ \
4251	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4252	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4254	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4255	}
4256
4257	AssertPtr(pVM);
4258	AssertPtr(pVCpu);
4259	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4260
4261	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4262	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4263	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4264	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4265	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4266	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4267	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4268	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4269	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4270
4271	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4272	#if 0
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4282	#endif
4283
4284	/* 32-bit guest-state fields. */
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4289	/* Unused 32-bit guest-state fields. */
4290	#if 0
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4294	#endif
4295
4296	/* Natural width guest-state fields. */
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4313	/* Unused natural width guest-state fields. */
4314	#if 0
4315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4317	#endif
4318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4320
4321	if (pVM->hm.s.fNestedPaging)
4322	{
4323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4325	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4326	}
4327	else
4328	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4329
4330	#undef VMXLOCAL_INIT_VMCS_SELREG
4331	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4332	return VINF_SUCCESS;
4333	}
4334
4335
4336	/**
4337	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4338	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4339	* darwin, running 64-bit guests).
4340	*
4341	* @returns VBox status code.
4342	* @param pVCpu Pointer to the VMCPU.
4343	* @param idxField The VMCS field encoding.
4344	* @param u64Val 16, 32 or 64 bits value.
4345	*/
4346	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4347	{
4348	int rc;
4349	switch (idxField)
4350	{
4351	/*
4352	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4353	*/
4354	/* 64-bit Control fields. */
4355	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4356	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4357	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4358	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4359	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4360	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4361	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4362	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4363	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4364	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4365	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4366	case VMX_VMCS64_CTRL_EPTP_FULL:
4367	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4368	/* 64-bit Read-only data fields. */
4369	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4370	/* 64-bit Guest-state fields. */
4371	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4372	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4373	case VMX_VMCS64_GUEST_PAT_FULL:
4374	case VMX_VMCS64_GUEST_EFER_FULL:
4375	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4376	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4377	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4378	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4379	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4380	/* 64-bit Host-state fields. */
4381	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4382	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4383	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4384	{
4385	rc = VMXWriteVmcs32(idxField, u64Val);
4386	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4387	break;
4388	}
4389
4390	/*
4391	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4392	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4393	*/
4394	/* Natural-width Guest-state fields. */
4395	case VMX_VMCS_GUEST_CR0:
4396	case VMX_VMCS_GUEST_CR3:
4397	case VMX_VMCS_GUEST_CR4:
4398	case VMX_VMCS_GUEST_ES_BASE:
4399	case VMX_VMCS_GUEST_CS_BASE:
4400	case VMX_VMCS_GUEST_SS_BASE:
4401	case VMX_VMCS_GUEST_DS_BASE:
4402	case VMX_VMCS_GUEST_FS_BASE:
4403	case VMX_VMCS_GUEST_GS_BASE:
4404	case VMX_VMCS_GUEST_LDTR_BASE:
4405	case VMX_VMCS_GUEST_TR_BASE:
4406	case VMX_VMCS_GUEST_GDTR_BASE:
4407	case VMX_VMCS_GUEST_IDTR_BASE:
4408	case VMX_VMCS_GUEST_DR7:
4409	case VMX_VMCS_GUEST_RSP:
4410	case VMX_VMCS_GUEST_RIP:
4411	case VMX_VMCS_GUEST_RFLAGS:
4412	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4413	case VMX_VMCS_GUEST_SYSENTER_ESP:
4414	case VMX_VMCS_GUEST_SYSENTER_EIP:
4415	{
4416	if ((u64Val >> 32ULL) == 0)
4417	{
4418	/* If this field is 64-bit, VT-x will zero out the top bits. */
4419	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4420	}
4421	else
4422	{
4423	/* Assert that only the 32->64 switcher case should ever come here. */
4424	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4425	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4426	}
4427	break;
4428	}
4429
4430	default:
4431	{
4432	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4433	rc = VERR_INVALID_PARAMETER;
4434	break;
4435	}
4436	}
4437	AssertRCReturn(rc, rc);
4438	return rc;
4439	}
4440
4441
4442	/**
4443	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4444	* hosts (except darwin) for 64-bit guests.
4445	*
4446	* @param pVCpu Pointer to the VMCPU.
4447	* @param idxField The VMCS field encoding.
4448	* @param u64Val 16, 32 or 64 bits value.
4449	*/
4450	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4451	{
4452	AssertPtr(pVCpu);
4453	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4454
4455	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4456	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4457
4458	/* Make sure there are no duplicates. */
4459	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4460	{
4461	if (pCache->Write.aField[i] == idxField)
4462	{
4463	pCache->Write.aFieldVal[i] = u64Val;
4464	return VINF_SUCCESS;
4465	}
4466	}
4467
4468	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4469	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4470	pCache->Write.cValidEntries++;
4471	return VINF_SUCCESS;
4472	}
4473
4474
4475	/**
4476	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4477	*
4478	* @param pVCpu Pointer to the VMCPU.
4479	* @param pCache Pointer to the VMCS cache.
4480	*
4481	* @remarks No-long-jump zone!!!
4482	*/
4483	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4484	{
4485	AssertPtr(pCache);
4486	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4487	{
4488	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4489	AssertRC(rc);
4490	}
4491	pCache->Write.cValidEntries = 0;
4492	}
4493
4494
4495	/**
4496	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4497	*
4498	* @param pVCpu Pointer to the VMCPU.
4499	* @param pCache Pointer to the VMCS cache.
4500	*
4501	* @remarks No-long-jump zone!!!
4502	*/
4503	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4504	{
4505	AssertPtr(pCache);
4506	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4507	{
4508	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4509	AssertRC(rc);
4510	}
4511	}
4512	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4513
4514
4515	/**
4516	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4517	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4518	* timer.
4519	*
4520	* @returns VBox status code.
4521	* @param pVM Pointer to the VM.
4522	* @param pVCpu Pointer to the VMCPU.
4523	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4524	* out-of-sync. Make sure to update the required fields
4525	* before using them.
4526	* @remarks No-long-jump zone!!!
4527	*/
4528	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4529	{
4530	int rc = VERR_INTERNAL_ERROR_5;
4531	bool fOffsettedTsc = false;
4532	if (pVM->hm.s.vmx.fUsePreemptTimer)
4533	{
4534	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4535
4536	/* Make sure the returned values have sane upper and lower boundaries. */
4537	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4538	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4539	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4540	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4541
4542	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4543	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4544	}
4545	else
4546	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4547
4548	if (fOffsettedTsc)
4549	{
4550	uint64_t u64CurTSC = ASMReadTSC();
4551	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4552	{
4553	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4554	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4555
4556	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4557	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4558	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4559	}
4560	else
4561	{
4562	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4563	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4564	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4565	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4566	}
4567	}
4568	else
4569	{
4570	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4571	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4572	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4573	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4574	}
4575	}
4576
4577
4578	/**
4579	* Determines if an exception is a benign exception. Benign exceptions
4580	* are ones which cannot cause double-faults.
4581	*
4582	* @returns true if the exception is benign, false otherwise.
4583	* @param uVector The exception vector.
4584	*/
4585	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4586	{
4587	switch (uVector)
4588	{
4589	case X86_XCPT_DB:
4590	case X86_XCPT_NMI:
4591	case X86_XCPT_BP:
4592	case X86_XCPT_OF:
4593	case X86_XCPT_BR:
4594	case X86_XCPT_UD:
4595	case X86_XCPT_NM:
4596	case X86_XCPT_CO_SEG_OVERRUN:
4597	case X86_XCPT_MF:
4598	case X86_XCPT_AC:
4599	case X86_XCPT_MC:
4600	case X86_XCPT_XF:
4601	return true;
4602	default:
4603	return false;
4604	}
4605	}
4606
4607
4608	/**
4609	* Determines if an exception is a contributory exception. Contributory
4610	* exceptions are ones which can cause double-faults.
4611	*
4612	* @returns true if the exception is contributory, false otherwise.
4613	* @param uVector The exception vector.
4614	*/
4615	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4616	{
4617	switch (uVector)
4618	{
4619	case X86_XCPT_GP:
4620	case X86_XCPT_SS:
4621	case X86_XCPT_NP:
4622	case X86_XCPT_TS:
4623	case X86_XCPT_DE:
4624	return true;
4625	default:
4626	return false;
4627	}
4628	return false;
4629	}
4630
4631
4632	/**
4633	* Determines if we are intercepting any contributory exceptions.
4634	*
4635	* @returns true if we are intercepting any contributory exception, false
4636	* otherwise.
4637	* @param pVCpu Pointer to the VMCPU.
4638	*/
4639	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4640	{
4641	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4642	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4643	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4644	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4645	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4646	{
4647	return true;
4648	}
4649	return false;
4650	}
4651
4652
4653	/**
4654	* Sets an event as a pending event to be injected into the guest.
4655	*
4656	* @param pVCpu Pointer to the VMCPU.
4657	* @param u32IntrInfo The VM-entry interruption-information field.
4658	* @param cbInstr The VM-entry instruction length in bytes (for software
4659	* interrupts, exceptions and privileged software
4660	* exceptions).
4661	* @param u32ErrCode The VM-entry exception error code.
4662	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4663	* page-fault.
4664	*/
4665	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4666	RTGCUINTPTR GCPtrFaultAddress)
4667	{
4668	Assert(!pVCpu->hm.s.Event.fPending);
4669	pVCpu->hm.s.Event.fPending = true;
4670	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4671	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4672	pVCpu->hm.s.Event.cbInstr = cbInstr;
4673	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4674	}
4675
4676
4677	/**
4678	* Handle a condition that occurred while delivering an event through the guest
4679	* IDT.
4680	*
4681	* @returns VBox status code (informational error codes included).
4682	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4683	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4684	* continue execution of the guest which will delivery the #DF.
4685	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4686	*
4687	* @param pVCpu Pointer to the VMCPU.
4688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4689	* out-of-sync. Make sure to update the required fields
4690	* before using them.
4691	* @param pVmxTransient Pointer to the VMX transient structure.
4692	*
4693	* @remarks No-long-jump zone!!!
4694	*/
4695	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4696	{
4697	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4698	AssertRCReturn(rc, rc);
4699	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4700	{
4701	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4702	AssertRCReturn(rc, rc);
4703
4704	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4705	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4706	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4707
4708	typedef enum
4709	{
4710	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4711	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4712	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4713	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4714	} VMXREFLECTXCPT;
4715
4716	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4717	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4718	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4719	{
4720	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4721	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4722	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4723	&& uExitVector == X86_XCPT_PF))
4724	{
4725	enmReflect = VMXREFLECTXCPT_XCPT;
4726	}
4727	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4728	&& uIdtVector == X86_XCPT_PF
4729	&& uExitVector == X86_XCPT_PF)
4730	{
4731	pVmxTransient->fVectoringPF = true;
4732	}
4733	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4734	&& hmR0VmxIsContributoryXcpt(uExitVector))
4735	{
4736	enmReflect = VMXREFLECTXCPT_DF;
4737	}
4738	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4739	&& uIdtVector == X86_XCPT_PF
4740	&& hmR0VmxIsContributoryXcpt(uExitVector))
4741	{
4742	enmReflect = VMXREFLECTXCPT_DF;
4743	}
4744	else if (uIdtVector == X86_XCPT_DF)
4745	enmReflect = VMXREFLECTXCPT_TF;
4746	}
4747	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4748	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4749	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4750	{
4751	/*
4752	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4753	* (whatever they are) as they reoccur when restarting the instruction.
4754	*/
4755	enmReflect = VMXREFLECTXCPT_XCPT;
4756	}
4757
4758	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4759	switch (enmReflect)
4760	{
4761	case VMXREFLECTXCPT_XCPT:
4762	{
4763	uint32_t u32ErrCode = 0;
4764	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4765	{
4766	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4767	AssertRCReturn(rc, rc);
4768	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4769	}
4770	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4771	0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4772	rc = VINF_SUCCESS;
4773	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4774	break;
4775	}
4776
4777	case VMXREFLECTXCPT_DF:
4778	{
4779	uint32_t u32IntrInfo;
4780	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4781	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4782	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4783	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4784	rc = VINF_VMX_DOUBLE_FAULT;
4785	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4786	break;
4787	}
4788
4789	case VMXREFLECTXCPT_TF:
4790	{
4791	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4792	rc = VINF_EM_RESET;
4793	break;
4794	}
4795
4796	default: /* shut up gcc. */
4797	break;
4798	}
4799	}
4800	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4801	return rc;
4802	}
4803
4804
4805	/**
4806	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4807	*
4808	* @returns VBox status code.
4809	* @param pVCpu Pointer to the VMCPU.
4810	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4811	* out-of-sync. Make sure to update the required fields
4812	* before using them.
4813	*
4814	* @remarks No-long-jump zone!!!
4815	*/
4816	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4817	{
4818	int rc = VINF_SUCCESS;
4819	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4820	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4821	{
4822	RTGCUINTREG uVal = 0;
4823	RTCCUINTREG uShadow = 0;
4824	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4825	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4826	AssertRCReturn(rc, rc);
4827	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4828	CPUMSetGuestCR0(pVCpu, uVal);
4829	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4830	}
4831	return rc;
4832	}
4833
4834
4835	/**
4836	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4837	*
4838	* @returns VBox status code.
4839	* @param pVCpu Pointer to the VMCPU.
4840	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4841	* out-of-sync. Make sure to update the required fields
4842	* before using them.
4843	*
4844	* @remarks No-long-jump zone!!!
4845	*/
4846	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4847	{
4848	int rc = VINF_SUCCESS;
4849	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4850	{
4851	RTGCUINTREG uVal = 0;
4852	RTCCUINTREG uShadow = 0;
4853	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4854	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4855	AssertRCReturn(rc, rc);
4856	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4857	CPUMSetGuestCR4(pVCpu, uVal);
4858	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4859	}
4860	return rc;
4861	}
4862
4863
4864	/**
4865	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4866	*
4867	* @returns VBox status code.
4868	* @param pVCpu Pointer to the VMCPU.
4869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4870	* out-of-sync. Make sure to update the required fields
4871	* before using them.
4872	*
4873	* @remarks No-long-jump zone!!!
4874	*/
4875	DECLINLINE(int) hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4876	{
4877	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4878	return VINF_SUCCESS;
4879
4880	RTGCUINTREG uVal = 0;
4881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4882	AssertRCReturn(rc, rc);
4883	pMixedCtx->rip = uVal;
4884	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4885	return rc;
4886	}
4887
4888
4889	/**
4890	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4891	*
4892	* @returns VBox status code.
4893	* @param pVCpu Pointer to the VMCPU.
4894	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4895	* out-of-sync. Make sure to update the required fields
4896	* before using them.
4897	*
4898	* @remarks No-long-jump zone!!!
4899	*/
4900	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4901	{
4902	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4903	return VINF_SUCCESS;
4904
4905	RTGCUINTREG uVal = 0;
4906	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4907	AssertRCReturn(rc, rc);
4908	pMixedCtx->rsp = uVal;
4909	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4910	return rc;
4911	}
4912
4913
4914	/**
4915	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4916	*
4917	* @returns VBox status code.
4918	* @param pVCpu Pointer to the VMCPU.
4919	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4920	* out-of-sync. Make sure to update the required fields
4921	* before using them.
4922	*
4923	* @remarks No-long-jump zone!!!
4924	*/
4925	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4926	{
4927	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4928	return VINF_SUCCESS;
4929
4930	RTGCUINTREG uVal = 0;
4931	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4932	AssertRCReturn(rc, rc);
4933	pMixedCtx->rflags.u64 = uVal;
4934
4935	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4936	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4937	{
4938	PVM pVM = pVCpu->CTX_SUFF(pVM);
4939	Assert(pVM->hm.s.vmx.pRealModeTSS);
4940	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4941	pMixedCtx->eflags.Bits.u1VM = 0;
4942	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4943	}
4944
4945	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4946	return rc;
4947	}
4948
4949
4950	/**
4951	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4952	* guest-CPU context.
4953	*/
4954	static int hmR0VmxSaveGuestGprs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4955	{
4956	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4957	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4958	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4959	return rc;
4960	}
4961
4962
4963	/**
4964	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4965	* from the guest-state area in the VMCS.
4966	*
4967	* @param pVCpu Pointer to the VMCPU.
4968	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4969	* out-of-sync. Make sure to update the required fields
4970	* before using them.
4971	*
4972	* @remarks No-long-jump zone!!!
4973	*/
4974	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4975	{
4976	uint32_t uIntrState = 0;
4977	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4978	AssertRC(rc);
4979
4980	if (!uIntrState)
4981	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4982	else
4983	{
4984	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4985	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4986	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4987	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4988	AssertRC(rc);
4989	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4990	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4991	}
4992	}
4993
4994
4995	/**
4996	* Saves the guest's activity state.
4997	*
4998	* @returns VBox status code.
4999	* @param pVCpu Pointer to the VMCPU.
5000	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5001	* out-of-sync. Make sure to update the required fields
5002	* before using them.
5003	*
5004	* @remarks No-long-jump zone!!!
5005	*/
5006	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5007	{
5008	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5009	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
5010	return VINF_SUCCESS;
5011	}
5012
5013
5014	/**
5015	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5016	* the current VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	int rc = VINF_SUCCESS;
5029	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5030	{
5031	uint32_t u32Val = 0;
5032	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5033	pMixedCtx->SysEnter.cs = u32Val;
5034	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5035	}
5036
5037	RTGCUINTREG uGCVal = 0;
5038	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5039	{
5040	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5041	pMixedCtx->SysEnter.eip = uGCVal;
5042	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5043	}
5044	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5045	{
5046	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5047	pMixedCtx->SysEnter.esp = uGCVal;
5048	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5049	}
5050	return rc;
5051	}
5052
5053
5054	/**
5055	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5056	* context.
5057	*
5058	* @returns VBox status code.
5059	* @param pVCpu Pointer to the VMCPU.
5060	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5061	* out-of-sync. Make sure to update the required fields
5062	* before using them.
5063	*
5064	* @remarks No-long-jump zone!!!
5065	*/
5066	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5067	{
5068	RTGCUINTREG uVal = 0;
5069	int rc = VINF_SUCCESS;
5070	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5071	{
5072	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5073	pMixedCtx->fs.u64Base = uVal;
5074	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5075	}
5076	return rc;
5077	}
5078
5079
5080	/**
5081	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5082	* context.
5083	*
5084	* @returns VBox status code.
5085	* @param pVCpu Pointer to the VMCPU.
5086	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087	* out-of-sync. Make sure to update the required fields
5088	* before using them.
5089	*
5090	* @remarks No-long-jump zone!!!
5091	*/
5092	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093	{
5094	RTGCUINTREG uVal = 0;
5095	int rc = VINF_SUCCESS;
5096	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5097	{
5098	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5099	pMixedCtx->gs.u64Base = uVal;
5100	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5101	}
5102	return rc;
5103	}
5104
5105
5106	/**
5107	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5108	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5109	*
5110	* @returns VBox status code.
5111	* @param pVCpu Pointer to the VMCPU.
5112	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5113	* out-of-sync. Make sure to update the required fields
5114	* before using them.
5115	*
5116	* @remarks No-long-jump zone!!!
5117	*/
5118	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5119	{
5120	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5121	return VINF_SUCCESS;
5122
5123	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5124	{
5125	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5126	pMsr += i;
5127	switch (pMsr->u32IndexMSR)
5128	{
5129	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5130	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5131	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5132	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5133	#if 0
5134	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5135	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5136	#endif
5137	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5138	default:
5139	{
5140	AssertFailed();
5141	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5142	}
5143	}
5144	}
5145	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5146	return VINF_SUCCESS;
5147	}
5148
5149
5150	/**
5151	* Saves the guest control registers from the current VMCS into the guest-CPU
5152	* context.
5153	*
5154	* @returns VBox status code.
5155	* @param pVCpu Pointer to the VMCPU.
5156	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5157	* out-of-sync. Make sure to update the required fields
5158	* before using them.
5159	*
5160	* @remarks No-long-jump zone!!!
5161	*/
5162	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5163	{
5164	RTGCUINTREG uVal = 0;
5165	RTGCUINTREG uShadow = 0;
5166	int rc = VINF_SUCCESS;
5167
5168	/* Guest CR0. Guest FPU. */
5169	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5170
5171	/* Guest CR4. */
5172	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5173	AssertRCReturn(rc, rc);
5174
5175	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5176	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5177	{
5178	PVM pVM = pVCpu->CTX_SUFF(pVM);
5179	if ( pVM->hm.s.fNestedPaging
5180	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5181	{
5182	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5183	if (pMixedCtx->cr3 != uVal)
5184	{
5185	CPUMSetGuestCR3(pVCpu, uVal);
5186	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5187	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5188	}
5189
5190	/* We require EFER to check PAE mode. */
5191	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5192
5193	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5194	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5195	{
5196	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5197	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5198	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5199	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5200	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5201	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5202	}
5203	AssertRCReturn(rc, rc);
5204	}
5205	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5206	}
5207	return rc;
5208	}
5209
5210
5211	/**
5212	* Reads a guest segment register from the current VMCS into the guest-CPU
5213	* context.
5214	*
5215	* @returns VBox status code.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param idxSel Index of the selector in the VMCS.
5218	* @param idxLimit Index of the segment limit in the VMCS.
5219	* @param idxBase Index of the segment base in the VMCS.
5220	* @param idxAccess Index of the access rights of the segment in the VMCS.
5221	* @param pSelReg Pointer to the segment selector.
5222	*
5223	* @remarks No-long-jump zone!!!
5224	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5225	* as that takes care of whether to read from the VMCS cache or not.
5226	*/
5227	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5228	PCPUMSELREG pSelReg)
5229	{
5230	uint32_t u32Val = 0;
5231	int rc = VMXReadVmcs32(idxSel, &u32Val);
5232	pSelReg->Sel = (uint16_t)u32Val;
5233	pSelReg->ValidSel = (uint16_t)u32Val;
5234	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5235
5236	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5237	pSelReg->u32Limit = u32Val;
5238
5239	RTGCUINTREG uGCVal = 0;
5240	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5241	pSelReg->u64Base = uGCVal;
5242
5243	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5244	pSelReg->Attr.u = u32Val;
5245	AssertRCReturn(rc, rc);
5246
5247	/*
5248	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5249	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5250	*/
5251	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5252	{
5253	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5254	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5255	}
5256	return rc;
5257	}
5258
5259
5260	/**
5261	* Saves the guest segment registers from the current VMCS into the guest-CPU
5262	* context.
5263	*
5264	* @returns VBox status code.
5265	* @param pVCpu Pointer to the VMCPU.
5266	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5267	* out-of-sync. Make sure to update the required fields
5268	* before using them.
5269	*
5270	* @remarks No-long-jump zone!!!
5271	*/
5272	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5273	{
5274	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5275	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5276	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5277	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5278	#else
5279	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5280	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5281	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5282	#endif
5283
5284	int rc = VINF_SUCCESS;
5285
5286	/* Guest segment registers. */
5287	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5288	{
5289	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5290	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5291	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5292	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5293	rc \|= VMXLOCAL_READ_SEG(ES, es);
5294	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5295	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5296	AssertRCReturn(rc, rc);
5297
5298	/* Restore segment attributes for real-on-v86 mode hack. */
5299	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5300	{
5301	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5302	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5303	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5304	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5305	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5306	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5307	}
5308	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5309	}
5310
5311	/* Guest LDTR. */
5312	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5313	{
5314	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5315	AssertRCReturn(rc, rc);
5316	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5317	}
5318
5319	/* Guest GDTR. */
5320	RTGCUINTREG uGCVal = 0;
5321	uint32_t u32Val = 0;
5322	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5323	{
5324	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5325	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5326	pMixedCtx->gdtr.pGdt = uGCVal;
5327	pMixedCtx->gdtr.cbGdt = u32Val;
5328	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5329	}
5330
5331	/* Guest IDTR. */
5332	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5333	{
5334	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5335	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5336	pMixedCtx->idtr.pIdt = uGCVal;
5337	pMixedCtx->idtr.cbIdt = u32Val;
5338	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5339	}
5340
5341	/* Guest TR. */
5342	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5343	{
5344	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5345
5346	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5347	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5348	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5349	AssertRCReturn(rc, rc);
5350	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5351	}
5352	return rc;
5353	}
5354
5355
5356	/**
5357	* Saves the guest debug registers from the current VMCS into the guest-CPU
5358	* context.
5359	*
5360	* @returns VBox status code.
5361	* @param pVCpu Pointer to the VMCPU.
5362	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5363	* out-of-sync. Make sure to update the required fields
5364	* before using them.
5365	*
5366	* @remarks No-long-jump zone!!!
5367	*/
5368	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5369	{
5370	int rc = VINF_SUCCESS;
5371	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5372	{
5373	RTGCUINTREG uVal;
5374	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5375	pMixedCtx->dr[7] = uVal;
5376
5377	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5378	}
5379	return rc;
5380	}
5381
5382
5383	/**
5384	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5385	*
5386	* @returns VBox status code.
5387	* @param pVCpu Pointer to the VMCPU.
5388	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5389	* out-of-sync. Make sure to update the required fields
5390	* before using them.
5391	*
5392	* @remarks No-long-jump zone!!!
5393	*/
5394	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5395	{
5396	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5397	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5398	return VINF_SUCCESS;
5399	}
5400
5401
5402	/**
5403	* Saves the entire guest state from the currently active VMCS into the
5404	* guest-CPU context. This essentially VMREADs all guest-data.
5405	*
5406	* @returns VBox status code.
5407	* @param pVCpu Pointer to the VMCPU.
5408	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5409	* out-of-sync. Make sure to update the required fields
5410	* before using them.
5411	*/
5412	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5413	{
5414	Assert(pVCpu);
5415	Assert(pMixedCtx);
5416
5417	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5418	return VINF_SUCCESS;
5419
5420	VMMRZCallRing3Disable(pVCpu);
5421
5422	int rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
5423	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5424
5425	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5427
5428	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5429	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5430
5431	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5433
5434	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5435	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5436
5437	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5438	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5439
5440	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5441	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5442
5443	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5444	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5445
5446	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5447	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5448
5449	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5451
5452	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5453	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5454
5455	VMMRZCallRing3Enable(pVCpu);
5456	return rc;
5457	}
5458
5459
5460	/**
5461	* Check per-VM and per-VCPU force flag actions that require us to go back to
5462	* ring-3 for one reason or another.
5463	*
5464	* @returns VBox status code (information status code included).
5465	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5466	* ring-3.
5467	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5468	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5469	* interrupts)
5470	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5471	* all EMTs to be in ring-3.
5472	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5473	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5474	* to the EM loop.
5475	*
5476	* @param pVM Pointer to the VM.
5477	* @param pVCpu Pointer to the VMCPU.
5478	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5479	* out-of-sync. Make sure to update the required fields
5480	* before using them.
5481	*/
5482	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5483	{
5484	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5485
5486	int rc = VERR_INTERNAL_ERROR_5;
5487	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5488	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5489	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5490	{
5491	/* We need the control registers now, make sure the guest-CPU context is updated. */
5492	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5493	AssertRCReturn(rc, rc);
5494
5495	/* Pending HM CR3 sync. */
5496	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5497	{
5498	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5499	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5500	}
5501	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5502	{
5503	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5504	AssertRC(rc);
5505	}
5506
5507	/* Pending PGM C3 sync. */
5508	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5509	{
5510	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5511	if (rc != VINF_SUCCESS)
5512	{
5513	AssertRC(rc);
5514	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5515	return rc;
5516	}
5517	}
5518
5519	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5520	/* -XXX- what was that about single stepping? */
5521	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5522	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5523	{
5524	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5525	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5526	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5527	return rc;
5528	}
5529
5530	/* Pending VM request packets, such as hardware interrupts. */
5531	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5532	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5533	{
5534	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5535	return VINF_EM_PENDING_REQUEST;
5536	}
5537
5538	/* Pending PGM pool flushes. */
5539	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5540	{
5541	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5542	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5543	}
5544
5545	/* Pending DMA requests. */
5546	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5547	{
5548	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5549	return VINF_EM_RAW_TO_R3;
5550	}
5551	}
5552
5553	/* Paranoia. */
5554	Assert(rc != VERR_EM_INTERPRETER);
5555	return VINF_SUCCESS;
5556	}
5557
5558
5559	/**
5560	* Converts any TRPM trap into a pending VMX event. This is typically used when
5561	* entering from ring-3 (not longjmp returns).
5562	*
5563	* @param pVCpu Pointer to the VMCPU.
5564	* @param pCtx Pointer to the guest-CPU context.
5565	*/
5566	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5567	{
5568	if (!TRPMHasTrap(pVCpu))
5569	return;
5570
5571	uint8_t uVector;
5572	TRPMEVENT enmTrpmEvent;
5573	RTGCUINT uErrCode;
5574	RTGCUINTPTR GCPtrFaultAddress;
5575	uint8_t cbInstr;
5576
5577	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5578	AssertRC(rc);
5579
5580	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5581	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5582	uint32_t u32ErrCode = uErrCode;
5583	if (enmTrpmEvent == TRPM_TRAP)
5584	{
5585	switch (uVector)
5586	{
5587	case X86_XCPT_BP:
5588	case X86_XCPT_OF:
5589	{
5590	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5591	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5592	break;
5593	}
5594
5595	case X86_XCPT_DF:
5596	case X86_XCPT_TS:
5597	case X86_XCPT_NP:
5598	case X86_XCPT_SS:
5599	case X86_XCPT_GP:
5600	case X86_XCPT_PF:
5601	case X86_XCPT_AC:
5602	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5603	them which VT-x/VMM pushes to the guest stack. */
5604	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5605	/* no break! */
5606	default:
5607	{
5608	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5609	if (uVector == X86_XCPT_PF)
5610	pCtx->cr2 = TRPMGetFaultAddress(pVCpu);
5611	break;
5612	}
5613	}
5614	}
5615	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5616	{
5617	if (uVector != X86_XCPT_NMI)
5618	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5619	else
5620	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5621	}
5622	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5623	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5624	else
5625	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5626
5627	rc = TRPMResetTrap(pVCpu);
5628	AssertRC(rc);
5629	Log(("Converted TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5630	u32IntrInfo, enmTrpmEvent, u32ErrCode, GCPtrFaultAddress));
5631	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5632	}
5633
5634
5635	/**
5636	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5637	* VT-x to execute any instruction.
5638	*
5639	* @param pvCpu Pointer to the VMCPU.
5640	*/
5641	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5642	{
5643	if (pVCpu->hm.s.Event.fPending)
5644	{
5645	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5646	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5647	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5648	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5649
5650	/* If a trap was already pending, we did something wrong! */
5651	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5652
5653	TRPMEVENT enmTrapType;
5654	switch (uVectorType)
5655	{
5656	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5657	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5658	enmTrapType = TRPM_HARDWARE_INT;
5659	break;
5660	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5661	enmTrapType = TRPM_SOFTWARE_INT;
5662	break;
5663	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5664	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5665	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5666	enmTrapType = TRPM_TRAP;
5667	break;
5668	default:
5669	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5670	enmTrapType = TRPM_32BIT_HACK;
5671	break;
5672	}
5673
5674	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5675	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5676	AssertRC(rc);
5677
5678	if (fErrorCodeValid)
5679	TRPMSetErrorCode(pVCpu, uErrorCode);
5680	/* A page-fault exception during a page-fault would become a double-fault. */
5681	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5682	&& uVector == X86_XCPT_PF)
5683	{
5684	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5685	}
5686	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5687	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT)
5688	{
5689	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5690	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5691	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5692	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5693	}
5694	pVCpu->hm.s.Event.fPending = false;
5695	}
5696	}
5697
5698
5699	/**
5700	* Does the necessary state syncing before doing a longjmp to ring-3.
5701	*
5702	* @param pVM Pointer to the VM.
5703	* @param pVCpu Pointer to the VMCPU.
5704	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5705	* out-of-sync. Make sure to update the required fields
5706	* before using them.
5707	* @param rcExit The reason for exiting to ring-3. Can be
5708	* VINF_VMM_UNKNOWN_RING3_CALL.
5709	*
5710	* @remarks No-long-jmp zone!!!
5711	*/
5712	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5713	{
5714	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5715
5716	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5717	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5718	AssertRC(rc);
5719
5720	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5721	if (CPUMIsGuestFPUStateActive(pVCpu))
5722	{
5723	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5724	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5725	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5726	}
5727
5728	/* Restore debug registers if necessary and resync on next R0 reentry. */
5729	if (CPUMIsGuestDebugStateActive(pVCpu))
5730	{
5731	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5732	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5733	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5734	}
5735	else if (CPUMIsHyperDebugStateActive(pVCpu))
5736	{
5737	CPUMR0LoadHostDebugState(pVM, pVCpu);
5738	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5739	}
5740
5741	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5742	}
5743
5744
5745	/**
5746	* An action requires us to go back to ring-3. This function does the necessary
5747	* steps before we can safely return to ring-3. This is not the same as longjmps
5748	* to ring-3, this is voluntary.
5749	*
5750	* @param pVM Pointer to the VM.
5751	* @param pVCpu Pointer to the VMCPU.
5752	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5753	* out-of-sync. Make sure to update the required fields
5754	* before using them.
5755	* @param rcExit The reason for exiting to ring-3. Can be
5756	* VINF_VMM_UNKNOWN_RING3_CALL.
5757	*/
5758	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5759	{
5760	Assert(pVM);
5761	Assert(pVCpu);
5762	Assert(pMixedCtx);
5763	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5764
5765	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5766	{
5767	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5768	return;
5769	}
5770	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5771	{
5772	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5773	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5774	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5775	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5776	return;
5777	}
5778
5779	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5780	VMMRZCallRing3Disable(pVCpu);
5781	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5782
5783	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5784	hmR0VmxUpdateTRPM(pVCpu);
5785
5786	/* Sync. the guest state. */
5787	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5788	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5789
5790	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5791	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5792	\| CPUM_CHANGED_LDTR
5793	\| CPUM_CHANGED_GDTR
5794	\| CPUM_CHANGED_IDTR
5795	\| CPUM_CHANGED_TR
5796	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5797
5798	/* On our way back from ring-3 the following needs to be done. */
5799	/** @todo This can change with preemption hooks. */
5800	if (rcExit == VINF_EM_RAW_INTERRUPT)
5801	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5802	else
5803	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5804
5805	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5806	VMMRZCallRing3Enable(pVCpu);
5807	}
5808
5809
5810	/**
5811	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5812	* longjump to ring-3 and possibly get preempted.
5813	*
5814	* @param pVCpu Pointer to the VMCPU.
5815	* @param enmOperation The operation causing the ring-3 longjump.
5816	* @param pvUser The user argument (pointer to the possibly
5817	* out-of-date guest-CPU context).
5818	*
5819	* @remarks Must never be called with @a enmOperation ==
5820	* VMMCALLRING3_VM_R0_ASSERTION.
5821	*/
5822	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5823	{
5824	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5825	Assert(pVCpu);
5826	Assert(pvUser);
5827	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5828	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5829
5830	VMMRZCallRing3Disable(pVCpu);
5831	Log(("hmR0VmxLongJmpToRing3\n"));
5832	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5833	VMMRZCallRing3Enable(pVCpu);
5834	}
5835
5836
5837	/**
5838	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5839	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5840	*
5841	* @returns VBox status code.
5842	* @param pVCpu Pointer to the VMCPU.
5843	*/
5844	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5845	{
5846	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5847	{
5848	/* Enable interrupt-window exiting if it's not in effect already. */
5849	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5850	{
5851	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5852	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5853	AssertRC(rc);
5854	}
5855	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5856	}
5857
5858
5859	/**
5860	* Injects any pending events into the guest if the guest is in a state to
5861	* receive them.
5862	*
5863	* @returns VBox status code (informational status codes included).
5864	* @param pVCpu Pointer to the VMCPU.
5865	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5866	* out-of-sync. Make sure to update the required fields
5867	* before using them.
5868	*/
5869	static int hmR0VmxInjectEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5870	{
5871	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5872	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5873	const bool fBlockMovSS = (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5874	const bool fBlockSti = (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5875
5876	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS));
5877	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5878
5879	int rc = VINF_SUCCESS;
5880	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5881	{
5882	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE((uint32_t)pVCpu->hm.s.Event.u64IntrInfo);
5883	bool fInject = true;
5884	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5885	{
5886	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5887	AssertRCReturn(rc, rc);
5888	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5889	if ( fBlockInt
5890	\|\| fBlockSti
5891	\|\| fBlockMovSS)
5892	{
5893	fInject = false;
5894	}
5895	}
5896	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5897	&& ( fBlockMovSS
5898	\|\| fBlockSti))
5899	{
5900	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5901	fInject = false;
5902	}
5903
5904	if (fInject)
5905	{
5906	Log(("Injecting pending event\n"));
5907	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5908	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5909	AssertRCReturn(rc, rc);
5910	pVCpu->hm.s.Event.fPending = false;
5911	}
5912	} /** @todo SMI. SMIs take priority over NMIs. */
5913	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5914	{
5915	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5916	if (!fBlockMovSS && !fBlockSti)
5917	{
5918	Log(("Injecting NMI\n"));
5919	RTGCUINTPTR uIntrInfo;
5920	uIntrInfo = X86_XCPT_NMI;
5921	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5922	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5923	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5924	AssertRCReturn(rc, rc);
5925	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5926	}
5927	else
5928	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5929	}
5930	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5931	{
5932	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5933	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5934	AssertRCReturn(rc, rc);
5935	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5936	if ( !fBlockInt
5937	&& !fBlockSti
5938	&& !fBlockMovSS)
5939	{
5940	uint8_t u8Interrupt = 0;
5941	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5942	if (RT_SUCCESS(rc))
5943	{
5944	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5945	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5946	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5947	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5948	}
5949	else
5950	{
5951	/** @todo Does this actually happen? If not turn it into an assertion. */
5952	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5953	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5954	rc = VINF_SUCCESS;
5955	}
5956	}
5957	else
5958	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5959	}
5960
5961	/*
5962	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5963	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5964	*/
5965	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5966	AssertRC(rc2);
5967	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5968	return rc;
5969	}
5970
5971
5972	/**
5973	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5974	*
5975	* @param pVCpu Pointer to the VMCPU.
5976	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5977	* out-of-sync. Make sure to update the required fields
5978	* before using them.
5979	*/
5980	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5981	{
5982	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5983	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5984	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5985	}
5986
5987
5988	/**
5989	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5990	*
5991	* @param pVCpu Pointer to the VMCPU.
5992	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5993	* out-of-sync. Make sure to update the required fields
5994	* before using them.
5995	*/
5996	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5997	{
5998	/* Inject the double-fault. */
5999	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6000	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6001	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6002	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6003	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6004	}
6005
6006
6007	/**
6008	* Injects a double-fault (#DF) exception into the VM.
6009	*
6010	* @returns VBox status code (informational status code included).
6011	* @param pVCpu Pointer to the VMCPU.
6012	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6013	* out-of-sync. Make sure to update the required fields
6014	* before using them.
6015	*/
6016	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6017	{
6018	/* Inject the double-fault. */
6019	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6020	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6021	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6022	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6023	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6024	}
6025
6026
6027	/**
6028	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6029	*
6030	* @param pVCpu Pointer to the VMCPU.
6031	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6032	* out-of-sync. Make sure to update the required fields
6033	* before using them.
6034	*/
6035	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6036	{
6037	/* Inject the debug-exception. */
6038	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6039	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6040	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6041	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6042	}
6043
6044
6045	/**
6046	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6047	*
6048	* @param pVCpu Pointer to the VMCPU.
6049	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6050	* out-of-sync. Make sure to update the required fields
6051	* before using them.
6052	* @param cbInstr The value of RIP that is to be pushed on the guest
6053	* stack.
6054	*/
6055	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6056	{
6057	/* Inject the overflow exception. */
6058	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6059	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6060	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6061	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6062	}
6063
6064
6065	/**
6066	* Injects a general-protection (#GP) fault into the VM.
6067	*
6068	* @returns VBox status code (informational status code included).
6069	* @param pVCpu Pointer to the VMCPU.
6070	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6071	* out-of-sync. Make sure to update the required fields
6072	* before using them.
6073	* @param u32ErrorCode The error code associated with the #GP.
6074	*/
6075	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6076	uint32_t *puIntrState)
6077	{
6078	/* Inject the general-protection fault. */
6079	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6080	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6081	if (fErrorCodeValid)
6082	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6083	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6084	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6085	}
6086
6087
6088	/**
6089	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6090	*
6091	* @param pVCpu Pointer to the VMCPU.
6092	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6093	* out-of-sync. Make sure to update the required fields
6094	* before using them.
6095	* @param uVector The software interrupt vector number.
6096	* @param cbInstr The value of RIP that is to be pushed on the guest
6097	* stack.
6098	*/
6099	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6100	{
6101	/* Inject the INTn. */
6102	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6103	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6104	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6105	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6106	}
6107
6108
6109	/**
6110	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6111	* stack.
6112	*
6113	* @returns VBox status code (information status code included).
6114	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6115	* @param pVM Pointer to the VM.
6116	* @param pMixedCtx Pointer to the guest-CPU context.
6117	* @param uValue The value to push to the guest stack.
6118	*/
6119	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6120	{
6121	/*
6122	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6123	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6124	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6125	*/
6126	if (pMixedCtx->sp == 1)
6127	return VINF_EM_RESET;
6128	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6129	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6130	AssertRCReturn(rc, rc);
6131	return rc;
6132	}
6133
6134
6135	/**
6136	* Injects an event into the guest upon VM-entry by updating the relevant fields
6137	* in the VM-entry area in the VMCS.
6138	*
6139	* @returns VBox status code (informational error codes included).
6140	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6141	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6142	*
6143	* @param pVCpu Pointer to the VMCPU.
6144	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6145	* out-of-sync. Make sure to update the required fields
6146	* before using them.
6147	* @param u64IntrInfo The VM-entry interruption-information field.
6148	* @param cbInstr The VM-entry instruction length in bytes (for software
6149	* interrupts, exceptions and privileged software
6150	* exceptions).
6151	* @param u32ErrCode The VM-entry exception error code.
6152	* @param puIntrState Pointer to the current guest interruptibility-state.
6153	* This interruptibility-state will be updated if
6154	* necessary. This cannot not be NULL.
6155	*
6156	* @remarks No-long-jump zone!!!
6157	* @remarks Requires CR0!
6158	*/
6159	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6160	uint32_t u32ErrCode, uint32_t *puIntrState)
6161	{
6162	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6163	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6164	Assert(puIntrState);
6165	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6166
6167	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6168	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6169
6170	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6171	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6172	\|\| !((*puIntrState) & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6173
6174	/* We require CR0 to check if the guest is in real-mode. */
6175	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6176	AssertRCReturn(rc, rc);
6177
6178	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6179
6180	/*
6181	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6182	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6183	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6184	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6185	*/
6186	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6187	{
6188	PVM pVM = pVCpu->CTX_SUFF(pVM);
6189	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6190	{
6191	Assert(PDMVmmDevHeapIsEnabled(pVM));
6192	Assert(pVM->hm.s.vmx.pRealModeTSS);
6193
6194	/* Save the required guest state bits from the VMCS. */
6195	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6196	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
6197	AssertRCReturn(rc, rc);
6198
6199	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6200	const size_t cbIdtEntry = 4;
6201	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6202	{
6203	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6204	if (uVector == X86_XCPT_DF)
6205	return VINF_EM_RESET;
6206	else if (uVector == X86_XCPT_GP)
6207	{
6208	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6209	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6210	}
6211
6212	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6213	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6214	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6215	}
6216
6217	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6218	uint16_t uGuestIp = pMixedCtx->ip;
6219	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6220	{
6221	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6222	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6223	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6224	}
6225	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6226	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6227
6228	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6229	uint16_t offIdtEntry = 0;
6230	RTSEL selIdtEntry = 0;
6231	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6232	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6233	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6234	AssertRCReturn(rc, rc);
6235
6236	/* Construct the stack frame for the interrupt/exception handler. */
6237	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6238	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6239	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6240	AssertRCReturn(rc, rc);
6241
6242	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6243	if (rc == VINF_SUCCESS)
6244	{
6245	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6246	pMixedCtx->rip = offIdtEntry;
6247	pMixedCtx->cs.Sel = selIdtEntry;
6248	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6249	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6250	\| HM_CHANGED_GUEST_RIP
6251	\| HM_CHANGED_GUEST_RFLAGS
6252	\| HM_CHANGED_GUEST_RSP;
6253
6254	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6255	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6256	{
6257	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6258	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6259	Log(("Clearing inhibition due to STI.\n"));
6260	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6261	}
6262	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6263	}
6264	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6265	return rc;
6266	}
6267	else
6268	{
6269	/*
6270	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6271	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6272	*/
6273	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6274	}
6275	}
6276
6277	/* Validate. */
6278	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6279	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6280	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6281
6282	/* Inject. */
6283	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6284	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6285	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6286	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6287	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6288	AssertRCReturn(rc, rc);
6289	return rc;
6290	}
6291
6292
6293	/**
6294	* Enters the VT-x session.
6295	*
6296	* @returns VBox status code.
6297	* @param pVM Pointer to the VM.
6298	* @param pVCpu Pointer to the VMCPU.
6299	* @param pCpu Pointer to the CPU info struct.
6300	*/
6301	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6302	{
6303	AssertPtr(pVM);
6304	AssertPtr(pVCpu);
6305	Assert(pVM->hm.s.vmx.fSupported);
6306	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6307	NOREF(pCpu);
6308
6309	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6310
6311	/* Make sure we're in VMX root mode. */
6312	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6313	if (!(u32HostCR4 & X86_CR4_VMXE))
6314	{
6315	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6316	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6317	}
6318
6319	/* Load the active VMCS as the current one. */
6320	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6321	if (RT_FAILURE(rc))
6322	return rc;
6323
6324	/** @todo this will change with preemption hooks where can can VMRESUME as long
6325	* as we're no preempted. */
6326	pVCpu->hm.s.fResumeVM = false;
6327	return VINF_SUCCESS;
6328	}
6329
6330
6331	/**
6332	* Leaves the VT-x session.
6333	*
6334	* @returns VBox status code.
6335	* @param pVM Pointer to the VM.
6336	* @param pVCpu Pointer to the VMCPU.
6337	* @param pCtx Pointer to the guest-CPU context.
6338	*/
6339	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6340	{
6341	AssertPtr(pVCpu);
6342	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6343	NOREF(pVM);
6344	NOREF(pCtx);
6345
6346	/** @todo this will change with preemption hooks where we only VMCLEAR when
6347	* we are actually going to be preempted, not all the time like we
6348	* currently do. */
6349	/*
6350	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6351	* and mark the VMCS launch-state as "clear".
6352	*/
6353	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6354	return rc;
6355	}
6356
6357
6358	/**
6359	* Saves the host state in the VMCS host-state.
6360	* Sets up the VM-exit MSR-load area.
6361	*
6362	* The CPU state will be loaded from these fields on every successful VM-exit.
6363	*
6364	* @returns VBox status code.
6365	* @param pVM Pointer to the VM.
6366	* @param pVCpu Pointer to the VMCPU.
6367	*
6368	* @remarks No-long-jump zone!!!
6369	*/
6370	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6371	{
6372	AssertPtr(pVM);
6373	AssertPtr(pVCpu);
6374	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6375
6376	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6377
6378	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6379	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6380	return VINF_SUCCESS;
6381
6382	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6384
6385	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6387
6388	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6389	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6390
6391	/* Reset the host-state-changed flag. */
6392	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6393	return rc;
6394	}
6395
6396
6397	/**
6398	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6399	* loaded from these fields on every successful VM-entry.
6400	*
6401	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6402	* Sets up the VM-entry controls.
6403	* Sets up the appropriate VMX non-root function to execute guest code based on
6404	* the guest CPU mode.
6405	*
6406	* @returns VBox status code.
6407	* @param pVM Pointer to the VM.
6408	* @param pVCpu Pointer to the VMCPU.
6409	* @param pCtx Pointer to the guest-CPU context.
6410	*
6411	* @remarks No-long-jump zone!!!
6412	*/
6413	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6414	{
6415	AssertPtr(pVM);
6416	AssertPtr(pVCpu);
6417	AssertPtr(pCtx);
6418	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6419
6420	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6421
6422	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6423
6424	/* Determine real-on-v86 mode. */
6425	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6426	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6427	&& CPUMIsGuestInRealModeEx(pCtx))
6428	{
6429	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6430	}
6431
6432	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6433	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6434
6435	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6436	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6437
6438	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6439	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6440
6441	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6442	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6443
6444	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6445	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6446
6447	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6448	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6449
6450	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6451	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6452
6453	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6455
6456	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6458
6459	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6461
6462	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6463	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6464	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6465
6466	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6467	return rc;
6468	}
6469
6470
6471	/**
6472	* Does the preparations before executing guest code in VT-x.
6473	*
6474	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6475	* recompiler. We must be cautious what we do here regarding committing
6476	* guest-state information into the the VMCS assuming we assuredly execute the
6477	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6478	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6479	* that the recompiler can (and should) use them when it resumes guest
6480	* execution. Otherwise such operations must be done when we can no longer
6481	* exit to ring-3.
6482	*
6483	* @returns VBox status code (informational status codes included).
6484	* @retval VINF_SUCCESS if we can proceed with running the guest.
6485	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6486	* into the guest.
6487	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6488	*
6489	* @param pVM Pointer to the VM.
6490	* @param pVCpu Pointer to the VMCPU.
6491	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6492	* out-of-sync. Make sure to update the required fields
6493	* before using them.
6494	* @param pVmxTransient Pointer to the VMX transient structure.
6495	*
6496	* @remarks Called with preemption disabled.
6497	*/
6498	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6499	{
6500	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6501
6502	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6503	PGMRZDynMapFlushAutoSet(pVCpu);
6504	#endif
6505
6506	/* Check force flag actions that might require us to go back to ring-3. */
6507	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6508	if (rc != VINF_SUCCESS)
6509	return rc;
6510
6511	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6512	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6513	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6514	{
6515	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6516	RTGCPHYS GCPhysApicBase;
6517	GCPhysApicBase = pMixedCtx->msrApicBase;
6518	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6519
6520	/* Unalias any existing mapping. */
6521	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6522	AssertRCReturn(rc, rc);
6523
6524	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6525	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6526	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6527	AssertRCReturn(rc, rc);
6528
6529	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6530	}
6531
6532	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6533	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6534	pVmxTransient->uEFlags = ASMIntDisableFlags();
6535	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6536	{
6537	ASMSetFlags(pVmxTransient->uEFlags);
6538	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6539	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6540	return VINF_EM_RAW_INTERRUPT;
6541	}
6542	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6543	#endif
6544
6545	/*
6546	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6547	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6548	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6549	*/
6550	rc = hmR0VmxInjectEvent(pVCpu, pMixedCtx);
6551	AssertRCReturn(rc, rc);
6552	return rc;
6553	}
6554
6555
6556	/**
6557	* Prepares to run guest code in VT-x and we've committed to doing so. This
6558	* means there is no backing out to ring-3 or anywhere else at this
6559	* point.
6560	*
6561	* @param pVM Pointer to the VM.
6562	* @param pVCpu Pointer to the VMCPU.
6563	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6564	* out-of-sync. Make sure to update the required fields
6565	* before using them.
6566	* @param pVmxTransient Pointer to the VMX transient structure.
6567	*
6568	* @remarks Called with preemption disabled.
6569	* @remarks No-long-jump zone!!!
6570	*/
6571	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6572	{
6573	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6574	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6575
6576	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6577	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6578	pVmxTransient->uEFlags = ASMIntDisableFlags();
6579	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6580	#endif
6581
6582	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6583	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6584	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6585	int rc = VINF_SUCCESS;
6586	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6587	{
6588	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6589	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6590	}
6591	else if (pVCpu->hm.s.fContextUseFlags)
6592	{
6593	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6594	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6595	}
6596	AssertRC(rc);
6597	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6598
6599	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6600	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6601	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6602
6603	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6604	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6605
6606	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6607	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6608	{
6609	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6610	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6611	}
6612
6613	/*
6614	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6615	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6616	*/
6617	if (pVM->hm.s.fTPRPatchingActive)
6618	{
6619	Assert(!CPUMIsGuestInLongMode(pVCpu));
6620
6621	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6622	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6623	AssertRC(rc);
6624
6625	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6626	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6627	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6628	see hmR0VmxLoadGuestApicState(). */
6629	}
6630
6631	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6632	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6633	to start executing. */
6634	}
6635
6636
6637	/**
6638	* Performs some essential restoration of state after running guest code in
6639	* VT-x.
6640	*
6641	* @param pVM Pointer to the VM.
6642	* @param pVCpu Pointer to the VMCPU.
6643	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6644	* out-of-sync. Make sure to update the required fields
6645	* before using them.
6646	* @param pVmxTransient Pointer to the VMX transient structure.
6647	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6648	*
6649	* @remarks Called with interrupts disabled.
6650	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6651	* unconditionally when it is safe to do so.
6652	*/
6653	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6654	{
6655	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6656	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6657
6658	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6659	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6660	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6661	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6662	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6663
6664	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6665	{
6666	/** @todo Find a way to fix hardcoding a guestimate. */
6667	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6668	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6669	}
6670
6671	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6672	Assert(!(ASMGetFlags() & X86_EFL_IF));
6673	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6674
6675	/* Restore the effects of TPR patching if any. */
6676	if (pVM->hm.s.fTPRPatchingActive)
6677	{
6678	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6679	AssertRC(rc);
6680	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6681	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6682	}
6683
6684	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6685	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6686
6687	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6688	uint32_t uExitReason;
6689	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6690	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6691	AssertRC(rc);
6692	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6693	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6694
6695	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6696	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6697
6698	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6699	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6700	{
6701	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6702	return;
6703	}
6704
6705	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6706	{
6707	/* Update the guest interruptibility-state from the VMCS. */
6708	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6709
6710	/*
6711	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6712	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6713	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6714	*/
6715	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6716	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6717	{
6718	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6719	AssertRC(rc);
6720	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6721	}
6722	}
6723	}
6724
6725
6726	/**
6727	* Runs the guest code using VT-x.
6728	*
6729	* @returns VBox status code.
6730	* @param pVM Pointer to the VM.
6731	* @param pVCpu Pointer to the VMCPU.
6732	* @param pCtx Pointer to the guest-CPU context.
6733	*
6734	* @remarks Called with preemption disabled.
6735	*/
6736	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6737	{
6738	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6739	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6740
6741	VMXTRANSIENT VmxTransient;
6742	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6743	int rc = VERR_INTERNAL_ERROR_5;
6744	uint32_t cLoops = 0;
6745	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6746
6747	for (;; cLoops++)
6748	{
6749	Assert(!HMR0SuspendPending());
6750	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6751	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6752	(unsigned)RTMpCpuId(), cLoops));
6753
6754	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6755	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6756	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6757	if (rc != VINF_SUCCESS)
6758	break;
6759
6760	/*
6761	* No longjmps to ring-3 from this point on!!!
6762	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6763	* This also disables flushing of the R0-logger instance (if any).
6764	*/
6765	VMMRZCallRing3Disable(pVCpu);
6766	VMMRZCallRing3RemoveNotification(pVCpu);
6767	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6768
6769	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6770	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6771
6772	/*
6773	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6774	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6775	*/
6776	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6777	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6778	{
6779	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6780	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6781	return rc;
6782	}
6783
6784	/* Handle the VM-exit. */
6785	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6786	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6787	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6788	rc = (*g_apfnVMExitHandlers[VmxTransient.uExitReason])(pVCpu, pCtx, &VmxTransient);
6789	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6790	if (rc != VINF_SUCCESS)
6791	break;
6792	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6793	{
6794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6795	rc = VINF_EM_RAW_INTERRUPT;
6796	break;
6797	}
6798	}
6799
6800	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6801	if (rc == VERR_EM_INTERPRETER)
6802	rc = VINF_EM_RAW_EMULATE_INSTR;
6803	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6804	return rc;
6805	}
6806
6807	#if 1
6808	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6809	{
6810	int rc;
6811	switch (rcReason)
6812	{
6813	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6814	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6815	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6820	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6846
6847	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6856
6857	case VMX_EXIT_VMCALL:
6858	case VMX_EXIT_VMCLEAR:
6859	case VMX_EXIT_VMLAUNCH:
6860	case VMX_EXIT_VMPTRLD:
6861	case VMX_EXIT_VMPTRST:
6862	case VMX_EXIT_VMREAD:
6863	case VMX_EXIT_VMRESUME:
6864	case VMX_EXIT_VMWRITE:
6865	case VMX_EXIT_VMXOFF:
6866	case VMX_EXIT_VMXON:
6867	case VMX_EXIT_INVEPT:
6868	case VMX_EXIT_INVVPID:
6869	case VMX_EXIT_VMFUNC:
6870	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6871	break;
6872	default:
6873	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6874	break;
6875	}
6876	return rc;
6877	}
6878	#endif
6879
6880	#ifdef DEBUG
6881	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6882	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6883	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6884	# define VMX_ASSERT_PREEMPT_CPUID() \
6885	do \
6886	{ \
6887	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6888	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6889	} while (0)
6890
6891	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6892	do { \
6893	AssertPtr(pVCpu); \
6894	AssertPtr(pMixedCtx); \
6895	AssertPtr(pVmxTransient); \
6896	Assert(pVmxTransient->fVMEntryFailed == false); \
6897	Assert(ASMIntAreEnabled()); \
6898	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6899	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6900	LogFunc(("\n")); \
6901	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6902	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6903	VMX_ASSERT_PREEMPT_CPUID(); \
6904	} while (0)
6905	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6906	do { \
6907	LogFunc(("\n")); \
6908	} while(0)
6909	#else /* Release builds */
6910	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6911	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6912	#endif
6913
6914
6915	/**
6916	* Advances the guest RIP after reading it from the VMCS.
6917	*
6918	* @returns VBox status code.
6919	* @param pVCpu Pointer to the VMCPU.
6920	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6921	* out-of-sync. Make sure to update the required fields
6922	* before using them.
6923	* @param pVmxTransient Pointer to the VMX transient structure.
6924	*
6925	* @remarks No-long-jump zone!!!
6926	*/
6927	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6928	{
6929	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6930	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6931	AssertRCReturn(rc, rc);
6932
6933	pMixedCtx->rip += pVmxTransient->cbInstr;
6934	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6935	return rc;
6936	}
6937
6938
6939	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6940	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6941	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6942	/**
6943	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6944	*/
6945	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6946	{
6947	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6949	return VINF_SUCCESS;
6950	}
6951
6952
6953	/**
6954	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6955	*/
6956	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6957	{
6958	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6959	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6960
6961	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6962	AssertRCReturn(rc, rc);
6963
6964	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6965	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6966	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6967
6968	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6969	{
6970	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6971	return VINF_EM_RAW_INTERRUPT;
6972	}
6973
6974	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6975	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6976	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6977	{
6978	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6979	return VINF_SUCCESS;
6980	}
6981	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6982	{
6983	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6984	return rc;
6985	}
6986
6987	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6988	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6989	switch (uIntrType)
6990	{
6991	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6992	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6993	/* no break */
6994	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6995	{
6996	switch (uVector)
6997	{
6998	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6999	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7000	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7001	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7002	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7003	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7004	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7005	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7006	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7007	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7008	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7009	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7010	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7011	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7012	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7013	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7014	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7015	#endif
7016	default:
7017	{
7018	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7019	AssertRCReturn(rc, rc);
7020
7021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7022	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7023	{
7024	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7025	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7026	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7027	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7028	AssertRCReturn(rc, rc);
7029	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7030	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7031	0 /* GCPtrFaultAddress */);
7032	AssertRCReturn(rc, rc);
7033	}
7034	else
7035	{
7036	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7037	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7038	}
7039	break;
7040	}
7041	}
7042	break;
7043	}
7044
7045	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7046	default:
7047	{
7048	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7049	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7050	break;
7051	}
7052	}
7053	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7054	return rc;
7055	}
7056
7057
7058	/**
7059	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7060	*/
7061	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7062	{
7063	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7064
7065	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7066	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7067	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7068	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7069	AssertRCReturn(rc, rc);
7070
7071	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7072	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7073	return VINF_SUCCESS;
7074	}
7075
7076
7077	/**
7078	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7079	*/
7080	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7081	{
7082	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7083	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7084	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7085	}
7086
7087
7088	/**
7089	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7090	*/
7091	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7092	{
7093	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7095	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7096	}
7097
7098
7099	/**
7100	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7101	*/
7102	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7103	{
7104	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7106	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7107	}
7108
7109
7110	/**
7111	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7112	*/
7113	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7114	{
7115	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7116	PVM pVM = pVCpu->CTX_SUFF(pVM);
7117	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7118	if (RT_LIKELY(rc == VINF_SUCCESS))
7119	{
7120	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7121	Assert(pVmxTransient->cbInstr == 2);
7122	}
7123	else
7124	{
7125	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7126	rc = VERR_EM_INTERPRETER;
7127	}
7128	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7129	return rc;
7130	}
7131
7132
7133	/**
7134	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7135	*/
7136	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7137	{
7138	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7139	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7140	AssertRCReturn(rc, rc);
7141
7142	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7143	return VINF_EM_RAW_EMULATE_INSTR;
7144
7145	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7146	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7147	}
7148
7149
7150	/**
7151	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7152	*/
7153	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7154	{
7155	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7156	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7157	AssertRCReturn(rc, rc);
7158
7159	PVM pVM = pVCpu->CTX_SUFF(pVM);
7160	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7161	if (RT_LIKELY(rc == VINF_SUCCESS))
7162	{
7163	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7164	Assert(pVmxTransient->cbInstr == 2);
7165	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7166	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7167	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7168	}
7169	else
7170	{
7171	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7172	rc = VERR_EM_INTERPRETER;
7173	}
7174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7175	return rc;
7176	}
7177
7178
7179	/**
7180	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7181	*/
7182	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7183	{
7184	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7185	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7186	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7187	AssertRCReturn(rc, rc);
7188
7189	PVM pVM = pVCpu->CTX_SUFF(pVM);
7190	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7191	if (RT_LIKELY(rc == VINF_SUCCESS))
7192	{
7193	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7194	Assert(pVmxTransient->cbInstr == 3);
7195	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7196	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7197	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7198	}
7199	else
7200	{
7201	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7202	rc = VERR_EM_INTERPRETER;
7203	}
7204	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7205	return rc;
7206	}
7207
7208
7209	/**
7210	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7211	*/
7212	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7213	{
7214	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7215	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7216	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7217	AssertRCReturn(rc, rc);
7218
7219	PVM pVM = pVCpu->CTX_SUFF(pVM);
7220	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7221	if (RT_LIKELY(rc == VINF_SUCCESS))
7222	{
7223	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7224	Assert(pVmxTransient->cbInstr == 2);
7225	}
7226	else
7227	{
7228	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7229	rc = VERR_EM_INTERPRETER;
7230	}
7231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7232	return rc;
7233	}
7234
7235
7236	/**
7237	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7238	*/
7239	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7240	{
7241	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7242	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7243	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7244	AssertRCReturn(rc, rc);
7245
7246	PVM pVM = pVCpu->CTX_SUFF(pVM);
7247	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7248	rc = VBOXSTRICTRC_VAL(rc2);
7249	if (RT_LIKELY(rc == VINF_SUCCESS))
7250	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7251	else
7252	{
7253	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7254	pVmxTransient->uExitQualification, rc));
7255	rc = VERR_EM_INTERPRETER;
7256	}
7257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7258	return rc;
7259	}
7260
7261
7262	/**
7263	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7264	*/
7265	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7266	{
7267	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7268	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7269	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7270	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7271	AssertRCReturn(rc, rc);
7272
7273	PVM pVM = pVCpu->CTX_SUFF(pVM);
7274	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7275	if (RT_LIKELY(rc == VINF_SUCCESS))
7276	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7277	else
7278	{
7279	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7280	rc = VERR_EM_INTERPRETER;
7281	}
7282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7283	return rc;
7284	}
7285
7286
7287	/**
7288	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7289	*/
7290	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7291	{
7292	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7293	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7294	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7295	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7296	AssertRCReturn(rc, rc);
7297
7298	PVM pVM = pVCpu->CTX_SUFF(pVM);
7299	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7300	rc = VBOXSTRICTRC_VAL(rc2);
7301	if (RT_LIKELY( rc == VINF_SUCCESS
7302	\|\| rc == VINF_EM_HALT))
7303	{
7304	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7305	AssertRCReturn(rc3, rc3);
7306
7307	if ( rc == VINF_EM_HALT
7308	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7309	{
7310	rc = VINF_SUCCESS;
7311	}
7312	}
7313	else
7314	{
7315	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7316	rc = VERR_EM_INTERPRETER;
7317	}
7318	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7319	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7321	return rc;
7322	}
7323
7324
7325	/**
7326	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7327	*/
7328	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7329	{
7330	/*
7331	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7332	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7333	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7334	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7335	*/
7336	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7337	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7338	}
7339
7340
7341	/**
7342	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7343	*/
7344	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7345	{
7346	/*
7347	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7348	* root operation. If we get there there is something funny going on.
7349	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7350	*/
7351	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7352	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7353	}
7354
7355
7356	/**
7357	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7358	*/
7359	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7360	{
7361	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7362	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7363	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7364	}
7365
7366
7367	/**
7368	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7369	*/
7370	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7371	{
7372	/*
7373	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7374	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7375	* See Intel spec. 25.3 "Other Causes of VM-exits".
7376	*/
7377	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7378	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7379	}
7380
7381
7382	/**
7383	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7384	* VM-exit.
7385	*/
7386	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7387	{
7388	/*
7389	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7390	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7391	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7392	*/
7393	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7394	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7395	}
7396
7397
7398	/**
7399	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7400	* VM-exit.
7401	*/
7402	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7403	{
7404	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7405	return VINF_EM_RESET;
7406	}
7407
7408
7409	/**
7410	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7411	*/
7412	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7413	{
7414	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7415	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7416	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7417	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7418	AssertRCReturn(rc, rc);
7419
7420	pMixedCtx->rip++;
7421	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7422	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7423	rc = VINF_SUCCESS;
7424	else
7425	rc = VINF_EM_HALT;
7426
7427	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7428	return rc;
7429	}
7430
7431
7432	/**
7433	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7434	*/
7435	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7436	{
7437	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7438	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7439	return VINF_SUCCESS;
7440	}
7441
7442
7443	/**
7444	* VM-exit handler for expiry of the VMX preemption timer.
7445	*/
7446	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7447	{
7448	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7449
7450	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7451	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7452	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7453
7454	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7455	PVM pVM = pVCpu->CTX_SUFF(pVM);
7456	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7457	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7458	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7459	}
7460
7461
7462	/**
7463	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7464	*/
7465	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7466	{
7467	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7468	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7469	/** @todo check if XSETBV is supported by the recompiler. */
7470	return VERR_EM_INTERPRETER;
7471	}
7472
7473
7474	/**
7475	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7476	*/
7477	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7478	{
7479	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7480	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7481	/** @todo implement EMInterpretInvpcid() */
7482	return VERR_EM_INTERPRETER;
7483	}
7484
7485
7486	/**
7487	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7488	* Error VM-exit.
7489	*/
7490	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7491	{
7492	uint32_t uIntrState;
7493	RTHCUINTREG uHCReg;
7494	uint64_t u64Val;
7495
7496	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7497	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7498	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7499	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7500	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7501	AssertRCReturn(rc, rc);
7502
7503	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7504	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7505	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7506	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7507
7508	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7509	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7510	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7511	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7512	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7513	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7514	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7515	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7516	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7517	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7518	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7519	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7520
7521	PVM pVM = pVCpu->CTX_SUFF(pVM);
7522	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7523
7524	return VERR_VMX_INVALID_GUEST_STATE;
7525	}
7526
7527
7528	/**
7529	* VM-exit handler for VM-entry failure due to an MSR-load
7530	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7531	*/
7532	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7533	{
7534	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7535	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7536	}
7537
7538
7539	/**
7540	* VM-exit handler for VM-entry failure due to a machine-check event
7541	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7542	*/
7543	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7544	{
7545	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7546	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7547	}
7548
7549
7550	/**
7551	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7552	* theory.
7553	*/
7554	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7555	{
7556	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7557	return VERR_VMX_UNDEFINED_EXIT_CODE;
7558	}
7559
7560
7561	/**
7562	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7563	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7564	* Conditional VM-exit.
7565	*/
7566	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7567	{
7568	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7569	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7571	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7572	return VERR_EM_INTERPRETER;
7573	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7574	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7575	}
7576
7577
7578	/**
7579	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7580	*/
7581	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7582	{
7583	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7584	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7586	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7587	return VERR_EM_INTERPRETER;
7588	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7589	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7590	}
7591
7592
7593	/**
7594	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7595	*/
7596	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7597	{
7598	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7599	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7600	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7601	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7602	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7603	AssertRCReturn(rc, rc);
7604
7605	PVM pVM = pVCpu->CTX_SUFF(pVM);
7606	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7607	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7608	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7610
7611	/* Update RIP and continue guest execution. */
7612	if (RT_LIKELY(rc == VINF_SUCCESS))
7613	{
7614	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7615	Assert(pVmxTransient->cbInstr == 2);
7616	}
7617	return rc;
7618	}
7619
7620
7621	/**
7622	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7623	*/
7624	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7625	{
7626	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7627	int rc = VINF_SUCCESS;
7628	PVM pVM = pVCpu->CTX_SUFF(pVM);
7629
7630	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7631	if ( pVM->hm.s.fTPRPatchingActive
7632	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7633	{
7634	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7635	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7636	{
7637	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7638	AssertRC(rc);
7639	}
7640
7641	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7642	Assert(pVmxTransient->cbInstr == 2);
7643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7644	return VINF_SUCCESS;
7645	}
7646
7647	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7648	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7649	{
7650	switch (pMixedCtx->ecx)
7651	{
7652	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7653	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7654	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7655	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7656	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7657	}
7658	}
7659	#ifdef VBOX_STRICT
7660	else
7661	{
7662	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7663	switch (pMixedCtx->ecx)
7664	{
7665	case MSR_IA32_SYSENTER_CS:
7666	case MSR_IA32_SYSENTER_EIP:
7667	case MSR_IA32_SYSENTER_ESP:
7668	case MSR_K8_FS_BASE:
7669	case MSR_K8_GS_BASE:
7670	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7671	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7672	case MSR_K8_LSTAR:
7673	case MSR_K6_STAR:
7674	case MSR_K8_SF_MASK:
7675	case MSR_K8_TSC_AUX:
7676	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7677	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7678	}
7679	}
7680	#endif
7681
7682	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7683	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7684	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7685	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7686	AssertRCReturn(rc, rc);
7687
7688	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7689	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7691
7692	/* Update guest-state and continue execution. */
7693	if (RT_LIKELY(rc == VINF_SUCCESS))
7694	{
7695	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7696
7697	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7698	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7699	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7700	{
7701	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7702	}
7703	}
7704	return rc;
7705	}
7706
7707
7708	/**
7709	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7710	*/
7711	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7712	{
7713	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7714	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7715	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7716	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7717	return VERR_EM_INTERPRETER;
7718	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7719	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7720	}
7721
7722
7723	/**
7724	* VM-exit handler for when the TPR value is lowered below the specified
7725	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7726	*/
7727	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7728	{
7729	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7730	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7731
7732	/*
7733	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7734	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7735	* resume guest execution.
7736	*/
7737	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7738	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7739	return VINF_SUCCESS;
7740	}
7741
7742
7743	/**
7744	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7745	* VM-exit.
7746	*
7747	* @retval VINF_SUCCESS when guest execution can continue.
7748	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7749	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7750	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7751	* recompiler.
7752	*/
7753	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7754	{
7755	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7756	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7757	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7758	AssertRCReturn(rc, rc);
7759
7760	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7761	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7762	PVM pVM = pVCpu->CTX_SUFF(pVM);
7763	switch (uAccessType)
7764	{
7765	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7766	{
7767	#if 0
7768	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7769	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7770	#else
7771	rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
7772	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7773	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7774	#endif
7775	AssertRCReturn(rc, rc);
7776
7777	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7778	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7779	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7780	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7781
7782	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7783	{
7784	case 0: /* CR0 */
7785	Log(("CR0 write rc=%d\n", rc));
7786	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7787	break;
7788	case 2: /* C2 **/
7789	/* Nothing to do here, CR2 it's not part of the VMCS. */
7790	break;
7791	case 3: /* CR3 */
7792	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7793	Log(("CR3 write rc=%d\n", rc));
7794	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7795	break;
7796	case 4: /* CR4 */
7797	Log(("CR4 write rc=%d\n", rc));
7798	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7799	break;
7800	case 8: /* CR8 */
7801	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7802	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7803	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7804	break;
7805	default:
7806	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7807	break;
7808	}
7809
7810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7811	break;
7812	}
7813
7814	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7815	{
7816	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7817	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7818	AssertRCReturn(rc, rc);
7819	Assert( !pVM->hm.s.fNestedPaging
7820	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7821	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7822
7823	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7824	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7825	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7826
7827	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7828	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7829	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7830	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7831	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7832	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7833	break;
7834	}
7835
7836	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7837	{
7838	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7839	AssertRCReturn(rc, rc);
7840	rc = EMInterpretCLTS(pVM, pVCpu);
7841	AssertRCReturn(rc, rc);
7842	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7843	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7844	Log(("CRX CLTS write rc=%d\n", rc));
7845	break;
7846	}
7847
7848	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7849	{
7850	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7851	AssertRCReturn(rc, rc);
7852	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7853	if (RT_LIKELY(rc == VINF_SUCCESS))
7854	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7856	Log(("CRX LMSW write rc=%d\n", rc));
7857	break;
7858	}
7859
7860	default:
7861	{
7862	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7863	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7864	}
7865	}
7866
7867	/* Validate possible error codes. */
7868	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7869	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7870	if (RT_SUCCESS(rc))
7871	{
7872	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7873	AssertRCReturn(rc2, rc2);
7874	}
7875
7876	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7877	return rc;
7878	}
7879
7880
7881	/**
7882	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7883	* VM-exit.
7884	*/
7885	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7886	{
7887	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7888	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7889
7890	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7891	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7892	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7893	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7894	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7895	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7896	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7897	AssertRCReturn(rc, rc);
7898
7899	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7900	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7901	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7902	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7903	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7904	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7905	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7906
7907	/* I/O operation lookup arrays. */
7908	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7909	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7910
7911	const uint32_t cbSize = s_aIOSize[uIOWidth];
7912	const uint32_t cbInstr = pVmxTransient->cbInstr;
7913	PVM pVM = pVCpu->CTX_SUFF(pVM);
7914	if (fIOString)
7915	{
7916	/* INS/OUTS - I/O String instruction. */
7917	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7918	/** @todo for now manually disassemble later optimize by getting the fields from
7919	* the VMCS. */
7920	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7921	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7922	* segment prefix info. */
7923	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7924	if (RT_SUCCESS(rc))
7925	{
7926	if (fIOWrite)
7927	{
7928	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7929	(DISCPUMODE)pDis->uAddrMode, cbSize);
7930	rc = VBOXSTRICTRC_VAL(rc2);
7931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7932	}
7933	else
7934	{
7935	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7936	(DISCPUMODE)pDis->uAddrMode, cbSize);
7937	rc = VBOXSTRICTRC_VAL(rc2);
7938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7939	}
7940	}
7941	else
7942	{
7943	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7944	rc = VINF_EM_RAW_EMULATE_INSTR;
7945	}
7946	}
7947	else
7948	{
7949	/* IN/OUT - I/O instruction. */
7950	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7951	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7952	if (fIOWrite)
7953	{
7954	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7955	rc = VBOXSTRICTRC_VAL(rc2);
7956	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7957	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7958	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7959	}
7960	else
7961	{
7962	uint32_t u32Result = 0;
7963	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7964	rc = VBOXSTRICTRC_VAL(rc2);
7965	if (IOM_SUCCESS(rc))
7966	{
7967	/* Save result of I/O IN instr. in AL/AX/EAX. */
7968	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7969	}
7970	else if (rc == VINF_IOM_R3_IOPORT_READ)
7971	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7973	}
7974	}
7975
7976	if (IOM_SUCCESS(rc))
7977	{
7978	pMixedCtx->rip += cbInstr;
7979	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7980	if (RT_LIKELY(rc == VINF_SUCCESS))
7981	{
7982	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7983	AssertRCReturn(rc, rc);
7984
7985	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7986	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7987	{
7988	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7989	for (unsigned i = 0; i < 4; i++)
7990	{
7991	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7992	if ( ( uIOPort >= pMixedCtx->dr[i]
7993	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7994	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7995	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7996	{
7997	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7998	uint64_t uDR6 = ASMGetDR6();
7999
8000	/* Clear all breakpoint status flags and set the one we just hit. */
8001	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8002	uDR6 \|= (uint64_t)RT_BIT(i);
8003
8004	/*
8005	* Note: AMD64 Architecture Programmer's Manual 13.1:
8006	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8007	* be cleared by software after the contents have been read.
8008	*/
8009	ASMSetDR6(uDR6);
8010
8011	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8012	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8013
8014	/* Paranoia. */
8015	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8016	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8017	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8018
8019	/* Resync DR7 */
8020	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8021	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8022
8023	/* Set #DB to be injected into the VM and continue guest execution. */
8024	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8025	break;
8026	}
8027	}
8028	}
8029	}
8030	}
8031
8032	#ifdef DEBUG
8033	if (rc == VINF_IOM_R3_IOPORT_READ)
8034	Assert(!fIOWrite);
8035	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8036	Assert(fIOWrite);
8037	else
8038	{
8039	AssertMsg( RT_FAILURE(rc)
8040	\|\| rc == VINF_SUCCESS
8041	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8042	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8043	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8044	}
8045	#endif
8046
8047	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8048	return rc;
8049	}
8050
8051
8052	/**
8053	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8054	* VM-exit.
8055	*/
8056	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8057	{
8058	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8059
8060	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8061	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8062	AssertRCReturn(rc, rc);
8063	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8064	{
8065	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8066	AssertRCReturn(rc, rc);
8067	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8068	{
8069	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8070	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8071	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8072	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8073	{
8074	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8075	pVCpu->hm.s.Event.fPending = true;
8076	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8077	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8078	AssertRCReturn(rc, rc);
8079	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
8080	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8081	else
8082	pVCpu->hm.s.Event.u32ErrCode = 0;
8083	}
8084	}
8085	}
8086	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8087	* emulation. */
8088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8089	return VERR_EM_INTERPRETER;
8090	}
8091
8092
8093	/**
8094	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8095	*/
8096	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8097	{
8098	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8099	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8100	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8101	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8102	AssertRCReturn(rc, rc);
8103	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8104	return VINF_EM_DBG_STOP;
8105	}
8106
8107
8108	/**
8109	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8110	*/
8111	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8112	{
8113	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8114	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8115
8116	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8117	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8118	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8119	return VINF_SUCCESS;
8120	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8121	return rc;
8122
8123	#if 0
8124	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8125	* just sync the whole thing. */
8126	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8127	#else
8128	/* Aggressive state sync. for now. */
8129	rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8130	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8131	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8132	#endif
8133	AssertRCReturn(rc, rc);
8134
8135	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8136	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8137	switch (uAccessType)
8138	{
8139	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8140	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8141	{
8142	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8143	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8144	{
8145	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8146	}
8147
8148	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8149	GCPhys &= PAGE_BASE_GC_MASK;
8150	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8151	PVM pVM = pVCpu->CTX_SUFF(pVM);
8152	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8153	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8154	CPUMCTX2CORE(pMixedCtx), GCPhys);
8155	rc = VBOXSTRICTRC_VAL(rc2);
8156	Log(("ApicAccess %RGp %#x rc=%d\n", GCPhys,
8157	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification), rc));
8158	if ( rc == VINF_SUCCESS
8159	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8160	\|\| rc == VERR_PAGE_NOT_PRESENT)
8161	{
8162	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8163	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8164	rc = VINF_SUCCESS;
8165	}
8166	break;
8167	}
8168
8169	default:
8170	rc = VINF_EM_RAW_EMULATE_INSTR;
8171	break;
8172	}
8173
8174	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8175	return rc;
8176	}
8177
8178
8179	/**
8180	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8181	* VM-exit.
8182	*/
8183	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8184	{
8185	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8186
8187	/* We should -not- get this VM-exit if the guest is debugging. */
8188	if (CPUMIsGuestDebugStateActive(pVCpu))
8189	{
8190	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8191	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8192	}
8193
8194	int rc = VERR_INTERNAL_ERROR_5;
8195	if ( !DBGFIsStepping(pVCpu)
8196	&& !CPUMIsHyperDebugStateActive(pVCpu))
8197	{
8198	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8199
8200	/* Don't intercept MOV DRx. */
8201	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8202	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8203	AssertRCReturn(rc, rc);
8204
8205	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8206	PVM pVM = pVCpu->CTX_SUFF(pVM);
8207	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8208	AssertRC(rc);
8209	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8210
8211	#ifdef VBOX_WITH_STATISTICS
8212	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8213	AssertRCReturn(rc, rc);
8214	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8216	else
8217	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8218	#endif
8219	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8220	return VINF_SUCCESS;
8221	}
8222
8223	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8224	* time and restore DRx registers afterwards */
8225	/*
8226	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8227	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8228	*/
8229	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8230	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8231	AssertRCReturn(rc, rc);
8232
8233	PVM pVM = pVCpu->CTX_SUFF(pVM);
8234	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8235	{
8236	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8237	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8238	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8239	if (RT_SUCCESS(rc))
8240	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8241	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8242	}
8243	else
8244	{
8245	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8246	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8247	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8249	}
8250
8251	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8252	if (RT_SUCCESS(rc))
8253	{
8254	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8255	AssertRCReturn(rc2, rc2);
8256	}
8257	return rc;
8258	}
8259
8260
8261	/**
8262	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8263	* Conditional VM-exit.
8264	*/
8265	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8266	{
8267	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8269
8270	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8271	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8272	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8273	return VINF_SUCCESS;
8274	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8275	return rc;
8276
8277	RTGCPHYS GCPhys = 0;
8278	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8279
8280	#if 0
8281	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8282	#else
8283	/* Aggressive state sync. for now. */
8284	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8285	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8286	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8287	#endif
8288	AssertRCReturn(rc, rc);
8289
8290	/*
8291	* If we succeed, resume guest execution.
8292	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8293	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8294	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8295	* weird case. See @bugref{6043}.
8296	*/
8297	PVM pVM = pVCpu->CTX_SUFF(pVM);
8298	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8299	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8300	rc = VBOXSTRICTRC_VAL(rc2);
8301	if ( rc == VINF_SUCCESS
8302	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8303	\|\| rc == VERR_PAGE_NOT_PRESENT)
8304	{
8305	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8306	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8307	return VINF_SUCCESS;
8308	}
8309	return rc;
8310	}
8311
8312
8313	/**
8314	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8315	* VM-exit.
8316	*/
8317	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8318	{
8319	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8320	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8321
8322	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8323	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8324	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8325	return VINF_SUCCESS;
8326	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8327	return rc;
8328
8329	RTGCPHYS GCPhys = 0;
8330	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8331	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8332	#if 0
8333	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8334	#else
8335	/* Aggressive state sync. for now. */
8336	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8337	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8338	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8339	#endif
8340	AssertRCReturn(rc, rc);
8341
8342	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8343	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8344
8345	RTGCUINT uErrorCode = 0;
8346	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8347	uErrorCode \|= X86_TRAP_PF_ID;
8348	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8349	uErrorCode \|= X86_TRAP_PF_RW;
8350	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8351	uErrorCode \|= X86_TRAP_PF_P;
8352
8353	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8354
8355	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8356	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8357
8358	/* Handle the pagefault trap for the nested shadow table. */
8359	PVM pVM = pVCpu->CTX_SUFF(pVM);
8360	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8361	TRPMResetTrap(pVCpu);
8362
8363	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8364	if ( rc == VINF_SUCCESS
8365	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8366	\|\| rc == VERR_PAGE_NOT_PRESENT)
8367	{
8368	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8369	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8370	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8371	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8372	return VINF_SUCCESS;
8373	}
8374
8375	Log(("EPT return to ring-3 rc=%d\n"));
8376	return rc;
8377	}
8378
8379
8380	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8381	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8382	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8383	/**
8384	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8385	*/
8386	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8387	{
8388	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8389
8390	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8391	AssertRCReturn(rc, rc);
8392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8393
8394	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8395	{
8396	/* Old-style FPU error reporting needs some extra work. */
8397	/** @todo don't fall back to the recompiler, but do it manually. */
8398	return VERR_EM_INTERPRETER;
8399	}
8400	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8401	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8402	return rc;
8403	}
8404
8405
8406	/**
8407	* VM-exit exception handler for #BP (Breakpoint exception).
8408	*/
8409	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8410	{
8411	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8412
8413	/** @todo Try optimize this by not saving the entire guest state unless
8414	* really needed. */
8415	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8416	AssertRCReturn(rc, rc);
8417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8418
8419	PVM pVM = pVCpu->CTX_SUFF(pVM);
8420	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8421	if (rc == VINF_EM_RAW_GUEST_TRAP)
8422	{
8423	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8424	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8425	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8426	AssertRCReturn(rc, rc);
8427
8428	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8429	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8430	}
8431
8432	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8433	return rc;
8434	}
8435
8436
8437	/**
8438	* VM-exit exception handler for #DB (Debug exception).
8439	*/
8440	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8441	{
8442	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8443
8444	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8445	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8446	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8447	AssertRCReturn(rc, rc);
8448
8449	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8450	uint64_t uDR6 = X86_DR6_INIT_VAL;
8451	uDR6 \|= (pVmxTransient->uExitQualification
8452	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8453	PVM pVM = pVCpu->CTX_SUFF(pVM);
8454	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8456	if (rc == VINF_EM_RAW_GUEST_TRAP)
8457	{
8458	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8459	pMixedCtx->dr[6] = uDR6;
8460
8461	if (CPUMIsGuestDebugStateActive(pVCpu))
8462	ASMSetDR6(pMixedCtx->dr[6]);
8463
8464	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8465	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8466
8467	/* Paranoia. */
8468	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8469	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8470	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8471
8472	/* Resync DR7. */
8473	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8474
8475	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8476	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8477	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8478	AssertRCReturn(rc,rc);
8479	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8480	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8481	return rc;
8482	}
8483	/* Return to ring 3 to deal with the debug exit code. */
8484	return rc;
8485	}
8486
8487
8488	/**
8489	* VM-exit exception handler for #NM (Device-not-available exception: floating
8490	* point exception).
8491	*/
8492	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8493	{
8494	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8495
8496	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8497	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8498	#endif
8499
8500	/* We require CR0 and EFER. EFER is always up-to-date. */
8501	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8502	AssertRCReturn(rc, rc);
8503
8504	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8505	PVM pVM = pVCpu->CTX_SUFF(pVM);
8506	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8507	if (rc == VINF_SUCCESS)
8508	{
8509	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8510	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8511	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8512	return VINF_SUCCESS;
8513	}
8514
8515	/* Forward #NM to the guest. */
8516	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8517	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8518	AssertRCReturn(rc, rc);
8519	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8520	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8521	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8522	return rc;
8523	}
8524
8525
8526	/**
8527	* VM-exit exception handler for #GP (General-protection exception).
8528	*
8529	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8530	*/
8531	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8532	{
8533	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8534
8535	int rc = VERR_INTERNAL_ERROR_5;
8536
8537	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8538	{
8539	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8540	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8541	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8542	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8543	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8544	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8545	AssertRCReturn(rc, rc);
8546	Log(("#GP Gst: RIP %#RX64\n", pMixedCtx->rip));
8547	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8548	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8549	return rc;
8550	#else
8551	/* We don't intercept #GP. */
8552	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8553	return VERR_VMX_UNEXPECTED_EXCEPTION;
8554	#endif
8555	}
8556
8557	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8558	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8559
8560	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8561	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8562	AssertRCReturn(rc, rc);
8563
8564	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8565	uint32_t cbOp = 0;
8566	PVM pVM = pVCpu->CTX_SUFF(pVM);
8567	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8568	if (RT_SUCCESS(rc))
8569	{
8570	rc = VINF_SUCCESS;
8571	Assert(cbOp == pDis->cbInstr);
8572	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8573	switch (pDis->pCurInstr->uOpcode)
8574	{
8575	case OP_CLI:
8576	pMixedCtx->eflags.Bits.u1IF = 0;
8577	pMixedCtx->rip += pDis->cbInstr;
8578	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8580	break;
8581
8582	case OP_STI:
8583	pMixedCtx->eflags.Bits.u1IF = 1;
8584	pMixedCtx->rip += pDis->cbInstr;
8585	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8586	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8587	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8588	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8589	break;
8590
8591	case OP_HLT:
8592	rc = VINF_EM_HALT;
8593	pMixedCtx->rip += pDis->cbInstr;
8594	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8596	break;
8597
8598	case OP_POPF:
8599	{
8600	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8601	uint32_t cbParm = 0;
8602	uint32_t uMask = 0;
8603	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8604	{
8605	cbParm = 4;
8606	uMask = 0xffffffff;
8607	}
8608	else
8609	{
8610	cbParm = 2;
8611	uMask = 0xffff;
8612	}
8613
8614	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8615	RTGCPTR GCPtrStack = 0;
8616	X86EFLAGS uEflags;
8617	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8618	&GCPtrStack);
8619	if (RT_SUCCESS(rc))
8620	{
8621	Assert(sizeof(uEflags.u32) >= cbParm);
8622	uEflags.u32 = 0;
8623	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8624	}
8625	if (RT_FAILURE(rc))
8626	{
8627	rc = VERR_EM_INTERPRETER;
8628	break;
8629	}
8630	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8631	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8632	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8633	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8634	pMixedCtx->eflags.Bits.u1RF = 0;
8635	pMixedCtx->esp += cbParm;
8636	pMixedCtx->esp &= uMask;
8637	pMixedCtx->rip += pDis->cbInstr;
8638	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8639	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8640	break;
8641	}
8642
8643	case OP_PUSHF:
8644	{
8645	uint32_t cbParm = 0;
8646	uint32_t uMask = 0;
8647	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8648	{
8649	cbParm = 4;
8650	uMask = 0xffffffff;
8651	}
8652	else
8653	{
8654	cbParm = 2;
8655	uMask = 0xffff;
8656	}
8657
8658	/* Get the stack pointer & push the contents of eflags onto the stack. */
8659	RTGCPTR GCPtrStack = 0;
8660	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8661	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8662	if (RT_FAILURE(rc))
8663	{
8664	rc = VERR_EM_INTERPRETER;
8665	break;
8666	}
8667	X86EFLAGS uEflags;
8668	uEflags = pMixedCtx->eflags;
8669	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8670	uEflags.Bits.u1RF = 0;
8671	uEflags.Bits.u1VM = 0;
8672
8673	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8674	if (RT_FAILURE(rc))
8675	{
8676	rc = VERR_EM_INTERPRETER;
8677	break;
8678	}
8679	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8680	pMixedCtx->esp -= cbParm;
8681	pMixedCtx->esp &= uMask;
8682	pMixedCtx->rip += pDis->cbInstr;
8683	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8685	break;
8686	}
8687
8688	case OP_IRET:
8689	{
8690	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8691	* instruction reference. */
8692	RTGCPTR GCPtrStack = 0;
8693	uint32_t uMask = 0xffff;
8694	uint16_t aIretFrame[3];
8695	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8696	{
8697	rc = VERR_EM_INTERPRETER;
8698	break;
8699	}
8700	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8701	&GCPtrStack);
8702	if (RT_SUCCESS(rc))
8703	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8704	if (RT_FAILURE(rc))
8705	{
8706	rc = VERR_EM_INTERPRETER;
8707	break;
8708	}
8709	pMixedCtx->eip = 0;
8710	pMixedCtx->ip = aIretFrame[0];
8711	pMixedCtx->cs.Sel = aIretFrame[1];
8712	pMixedCtx->cs.ValidSel = aIretFrame[1];
8713	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8714	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8715	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8716	pMixedCtx->sp += sizeof(aIretFrame);
8717	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8718	\| HM_CHANGED_GUEST_RFLAGS;
8719	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8720	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8721	break;
8722	}
8723
8724	case OP_INT:
8725	{
8726	uint16_t uVector = pDis->Param1.uValue & 0xff;
8727	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8728	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8729	break;
8730	}
8731
8732	case OP_INTO:
8733	{
8734	if (pMixedCtx->eflags.Bits.u1OF)
8735	{
8736	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8738	}
8739	break;
8740	}
8741
8742	default:
8743	{
8744	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8745	EMCODETYPE_SUPERVISOR);
8746	rc = VBOXSTRICTRC_VAL(rc2);
8747	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8748	Log2(("#GP rc=%Rrc\n", rc));
8749	break;
8750	}
8751	}
8752	}
8753	else
8754	rc = VERR_EM_INTERPRETER;
8755
8756	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8757	("#GP Unexpected rc=%Rrc\n", rc));
8758	return rc;
8759	}
8760
8761
8762	/**
8763	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8764	* the exception reported in the VMX transient structure back into the VM.
8765	*
8766	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8767	* VMX transient structure to be up-to-date.
8768	*/
8769	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8770	{
8771	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8772
8773	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8774	hmR0VmxCheckExitDueToEventDelivery(). */
8775	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8776	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8777	return VINF_SUCCESS;
8778	}
8779
8780
8781	/**
8782	* VM-exit exception handler for #PF (Page-fault exception).
8783	*/
8784	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8785	{
8786	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8787	PVM pVM = pVCpu->CTX_SUFF(pVM);
8788	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8789	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8790	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8791	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8792	AssertRCReturn(rc, rc);
8793
8794	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8795	if (pVM->hm.s.fNestedPaging)
8796	{
8797	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8798	{
8799	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8800	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8801	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8802	}
8803	else
8804	{
8805	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8806	rc = hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8807	}
8808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8809	return rc;
8810	}
8811	#else
8812	Assert(!pVM->hm.s.fNestedPaging);
8813	#endif
8814
8815	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8816	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8817	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8818	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8819	AssertRCReturn(rc, rc);
8820	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8821	if ( pVM->hm.s.fTRPPatchingAllowed
8822	&& pVM->hm.s.pGuestPatchMem
8823	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8824	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8825	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8826	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8827	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8828	{
8829	RTGCPHYS GCPhys;
8830	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8831	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8832	if ( rc == VINF_SUCCESS
8833	&& GCPhys == GCPhysApicBase)
8834	{
8835	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8836	AssertRCReturn(rc, rc);
8837
8838	/* Only attempt to patch the instruction once. */
8839	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8840	if (!pPatch)
8841	return VINF_EM_HM_PATCH_TPR_INSTR;
8842	}
8843	}
8844	#endif
8845
8846	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8847	AssertRCReturn(rc, rc);
8848
8849	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8850
8851	/* Forward it to the trap handler first. */
8852	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8853	(RTGCPTR)pVmxTransient->uExitQualification);
8854
8855	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8856	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8857
8858	if (rc == VINF_SUCCESS)
8859	{
8860	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8861	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8862	* memory? We don't update the whole state here... */
8863	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8864	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8865	TRPMResetTrap(pVCpu);
8866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8867	return rc;
8868	}
8869	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8870	{
8871	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8872	{
8873	/* It's a guest page fault and needs to be reflected to the guest. */
8874	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8875	TRPMResetTrap(pVCpu);
8876	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8877	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8878	pVmxTransient->cbInstr, uGstErrorCode, pMixedCtx->cr2);
8879	}
8880	else
8881	{
8882	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8883	TRPMResetTrap(pVCpu);
8884	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8885	Log(("#PF: Pending #DF injection\n"));
8886	}
8887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8888	return VINF_SUCCESS;
8889	}
8890
8891	TRPMResetTrap(pVCpu);
8892	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8893	return rc;
8894	}
8895

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45542

Download in other formats: