HMVMXR0.cpp@ 45409

Last change on this file since 45409 was 45409, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits. Save preemption timer value on every VM-exit and reset it up when required.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 360.0 KB

Line
1	/* $Id: HMVMXR0.cpp 45409 2013-04-08 15:01:06Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205
206	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
207	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
208	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257
258	/*******************************************************************************
259	* Global Variables *
260	*******************************************************************************/
261	/** @todo Move this to hm_vmx.h. */
262	/**
263	* VM-exit handler.
264	*
265	* @returns VBox status code.
266	* @param pVM Pointer to the VM.
267	* @param pVCpu Pointer to the VMCPU.
268	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
269	* out-of-sync. Make sure to update the required
270	* fields before using them.
271	* @param pVmxTransient Pointer to the VMX-transient structure.
272	*/
273	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	/** Pointer to VM-exit handler. */
275	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
276
277	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
278	{
279	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
280	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
281	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
282	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
283	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
284	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
285	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
286	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
287	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
288	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
289	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
290	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
291	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
292	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
293	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
294	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
295	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
296	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
297	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
298	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
299	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
300	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
301	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
302	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
303	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
304	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
305	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
306	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
307	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
308	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
309	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
310	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
311	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
312	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
313	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
314	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
315	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
316	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
317	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
318	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
319	/* 40 UNDEFINED */ hmR0VmxExitPause,
320	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
321	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
322	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
323	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
324	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
326	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
327	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
328	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
329	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
330	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
331	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
332	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
333	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
334	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
335	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
336	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
337	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
338	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
339	};
340
341	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
342	{
343	/* 0 */ "(Not Used)",
344	/* 1 */ "VMCALL executed in VMX root operation.",
345	/* 2 */ "VMCLEAR with invalid physical address.",
346	/* 3 */ "VMCLEAR with VMXON pointer.",
347	/* 4 */ "VMLAUNCH with non-clear VMCS.",
348	/* 5 */ "VMRESUME with non-launched VMCS.",
349	/* 6 */ "VMRESUME after VMXOFF",
350	/* 7 */ "VM entry with invalid control fields.",
351	/* 8 */ "VM entry with invalid host state fields.",
352	/* 9 */ "VMPTRLD with invalid physical address.",
353	/* 10 */ "VMPTRLD with VMXON pointer.",
354	/* 11 */ "VMPTRLD with incorrect revision identifier.",
355	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
356	/* 13 */ "VMWRITE to read-only VMCS component.",
357	/* 14 */ "(Not Used)",
358	/* 15 */ "VMXON executed in VMX root operation.",
359	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
360	/* 17 */ "VM entry with non-launched executing VMCS.",
361	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
362	/* 19 */ "VMCALL with non-clear VMCS.",
363	/* 20 */ "VMCALL with invalid VM-exit control fields.",
364	/* 21 */ "(Not Used)",
365	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
366	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
367	/* 24 */ "VMCALL with invalid SMM-monitor features.",
368	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
369	/* 26 */ "VM entry with events blocked by MOV SS.",
370	/* 27 */ "(Not Used)",
371	/* 28 */ "Invalid operand to INVEPT/INVVPID."
372	};
373
374
375	/**
376	* Updates the VM's last error record. If there was a VMX instruction error,
377	* reads the error data from the VMCS and updates VCPU's last error record as
378	* well.
379	*
380	* @param pVM Pointer to the VM.
381	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
382	* VERR_VMX_GENERIC).
383	* @param rc The error code.
384	*/
385	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
386	{
387	AssertPtr(pVM);
388	if (rc == VERR_VMX_GENERIC)
389	{
390	AssertPtrReturnVoid(pVCpu);
391	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
392	}
393	pVM->hm.s.lLastError = rc;
394	}
395
396
397	/**
398	* Reads the VM-entry interruption-information field from the VMCS into the VMX
399	* transient structure.
400	*
401	* @returns VBox status code.
402	* @param pVmxTransient Pointer to the VMX transient structure.
403	*
404	* @remarks No-long-jump zone!!!
405	*/
406	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
407	{
408	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
409	AssertRCReturn(rc, rc);
410	return VINF_SUCCESS;
411	}
412
413
414	/**
415	* Reads the VM-entry exception error code field from the VMCS into
416	* the VMX transient structure.
417	*
418	* @returns VBox status code.
419	* @param pVmxTransient Pointer to the VMX transient structure.
420	*
421	* @remarks No-long-jump zone!!!
422	*/
423	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
424	{
425	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
426	AssertRCReturn(rc, rc);
427	return VINF_SUCCESS;
428	}
429
430
431	/**
432	* Reads the VM-entry exception error code field from the VMCS into
433	* the VMX transient structure.
434	*
435	* @returns VBox status code.
436	* @param pVmxTransient Pointer to the VMX transient structure.
437	*
438	* @remarks No-long-jump zone!!!
439	*/
440	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
441	{
442	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
443	AssertRCReturn(rc, rc);
444	return VINF_SUCCESS;
445	}
446
447
448	/**
449	* Reads the VM-exit interruption-information field from the VMCS into the VMX
450	* transient structure.
451	*
452	* @returns VBox status code.
453	* @param pVmxTransient Pointer to the VMX transient structure.
454	*/
455	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
456	{
457	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
458	{
459	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
460	AssertRCReturn(rc, rc);
461	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
462	}
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Reads the VM-exit interruption error code from the VMCS into the VMX
469	* transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVmxTransient Pointer to the VMX transient structure.
473	*/
474	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
477	{
478	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
479	AssertRCReturn(rc, rc);
480	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
481	}
482	return VINF_SUCCESS;
483	}
484
485
486	/**
487	* Reads the VM-exit instruction length field from the VMCS into the VMX
488	* transient structure.
489	*
490	* @returns VBox status code.
491	* @param pVmxTransient Pointer to the VMX transient structure.
492	*/
493	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
494	{
495	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
496	{
497	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
498	AssertRCReturn(rc, rc);
499	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
500	}
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the exit qualification from the VMCS into the VMX transient structure.
507	*
508	* @returns VBox status code.
509	* @param pVmxTransient Pointer to the VMX transient structure.
510	*/
511	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
512	{
513	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
514	{
515	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
516	AssertRCReturn(rc, rc);
517	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
518	}
519	return VINF_SUCCESS;
520	}
521
522
523	/**
524	* Reads the IDT-vectoring information field from the VMCS into the VMX
525	* transient structure.
526	*
527	* @returns VBox status code.
528	* @param pVmxTransient Pointer to the VMX transient structure.
529	*
530	* @remarks No-long-jump zone!!!
531	*/
532	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
533	{
534	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
535	{
536	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
537	AssertRCReturn(rc, rc);
538	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
539	}
540	return VINF_SUCCESS;
541	}
542
543
544	/**
545	* Reads the IDT-vectoring error code from the VMCS into the VMX
546	* transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVmxTransient Pointer to the VMX transient structure.
550	*/
551	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
552	{
553	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
556	AssertRCReturn(rc, rc);
557	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
558	}
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Enters VMX root mode operation on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pVM Pointer to the VM (optional, can be NULL, after
568	* a resume).
569	* @param HCPhysCpuPage Physical address of the VMXON region.
570	* @param pvCpuPage Pointer to the VMXON region.
571	*/
572	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
573	{
574	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
575	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
576
577	if (pVM)
578	{
579	/* Write the VMCS revision dword to the VMXON region. */
580	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
581	}
582
583	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
584	RTCCUINTREG fFlags = ASMIntDisableFlags();
585
586	/* Enable the VMX bit in CR4. */
587	RTCCUINTREG uCr4 = ASMGetCR4();
588	if (!(uCr4 & X86_CR4_VMXE))
589	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
590
591	/* Enter VMXON root mode. */
592	int rc = VMXEnable(HCPhysCpuPage);
593
594	/* Restore interrupts. */
595	ASMSetFlags(fFlags);
596	return rc;
597	}
598
599
600	/**
601	* Exits VMX root mode operation on the current CPU.
602	*
603	* @returns VBox status code.
604	*/
605	static int hmR0VmxLeaveRootMode(void)
606	{
607	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
608	RTCCUINTREG fFlags = ASMIntDisableFlags();
609	int rc = VINF_SUCCESS;
610
611	/* If we're for some reason not in VMX root mode, then don't leave it. */
612	if (ASMGetCR4() & X86_CR4_VMXE)
613	{
614	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
615	VMXDisable();
616	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
617	}
618	else
619	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
620
621	/* Restore interrupts. */
622	ASMSetFlags(fFlags);
623	return rc;
624	}
625
626
627	/**
628	* Allocates and maps one physically contiguous page. The allocated page is
629	* zero'd out. (Used by various VT-x structures).
630	*
631	* @returns IPRT status code.
632	* @param pMemObj Pointer to the ring-0 memory object.
633	* @param ppVirt Where to store the virtual address of the
634	* allocation.
635	* @param pPhys Where to store the physical address of the
636	* allocation.
637	*/
638	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
639	{
640	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
641	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
642	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
643
644	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
645	if (RT_FAILURE(rc))
646	return rc;
647	ppVirt = RTR0MemObjAddress(pMemObj);
648	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
649	ASMMemZero32(*ppVirt, PAGE_SIZE);
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Frees and unmaps an allocated physical page.
656	*
657	* @param pMemObj Pointer to the ring-0 memory object.
658	* @param ppVirt Where to re-initialize the virtual address of
659	* allocation as 0.
660	* @param pHCPhys Where to re-initialize the physical address of the
661	* allocation as 0.
662	*/
663	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
664	{
665	AssertPtr(pMemObj);
666	AssertPtr(ppVirt);
667	AssertPtr(pHCPhys);
668	if (*pMemObj != NIL_RTR0MEMOBJ)
669	{
670	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
671	AssertRC(rc);
672	*pMemObj = NIL_RTR0MEMOBJ;
673	*ppVirt = 0;
674	*pHCPhys = 0;
675	}
676	}
677
678
679	/**
680	* Worker function to free VT-x related structures.
681	*
682	* @returns IPRT status code.
683	* @param pVM Pointer to the VM.
684	*/
685	static void hmR0VmxStructsFree(PVM pVM)
686	{
687	for (VMCPUID i = 0; i < pVM->cCpus; i++)
688	{
689	PVMCPU pVCpu = &pVM->aCpus[i];
690	AssertPtr(pVCpu);
691
692	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
693	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
694
695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
696	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
697
698	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
699	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
700	}
701
702	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
703	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
704	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
705	#endif
706	}
707
708
709	/**
710	* Worker function to allocate VT-x related VM structures.
711	*
712	* @returns IPRT status code.
713	* @param pVM Pointer to the VM.
714	*/
715	static int hmR0VmxStructsAlloc(PVM pVM)
716	{
717	/*
718	* Initialize members up-front so we can cleanup properly on allocation failure.
719	*/
720	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
721	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
722	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
723	pVM->hm.s.vmx.HCPhys##a_Name = 0;
724
725	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
726	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
727	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
728	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
729
730	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
731	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
732	#endif
733	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
734
735	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
736	for (VMCPUID i = 0; i < pVM->cCpus; i++)
737	{
738	PVMCPU pVCpu = &pVM->aCpus[i];
739	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
740	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
741	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
742	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
743	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
744	}
745	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
746	#undef VMXLOCAL_INIT_VM_MEMOBJ
747
748	/*
749	* Allocate all the VT-x structures.
750	*/
751	int rc = VINF_SUCCESS;
752	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
753	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
754	if (RT_FAILURE(rc))
755	goto cleanup;
756	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
757	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
758	#endif
759
760	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
762	{
763	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
764	&pVM->hm.s.vmx.HCPhysApicAccess);
765	if (RT_FAILURE(rc))
766	goto cleanup;
767	}
768
769	/*
770	* Initialize per-VCPU VT-x structures.
771	*/
772	for (VMCPUID i =0; i < pVM->cCpus; i++)
773	{
774	PVMCPU pVCpu = &pVM->aCpus[i];
775	AssertPtr(pVCpu);
776
777	/* Allocate the VM control structure (VMCS). */
778	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
779	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
780	if (RT_FAILURE(rc))
781	goto cleanup;
782
783	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
784	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
785	{
786	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
787	&pVCpu->hm.s.vmx.HCPhysVirtApic);
788	if (RT_FAILURE(rc))
789	goto cleanup;
790	}
791
792	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
793	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
794	{
795	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
796	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
797	if (RT_FAILURE(rc))
798	goto cleanup;
799	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
800	}
801
802	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
803	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
804	if (RT_FAILURE(rc))
805	goto cleanup;
806
807	/* Allocate the VM-exit MSR-load page for the host MSRs. */
808	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
809	if (RT_FAILURE(rc))
810	goto cleanup;
811	}
812
813	return VINF_SUCCESS;
814
815	cleanup:
816	hmR0VmxStructsFree(pVM);
817	return rc;
818	}
819
820
821	/**
822	* Does global VT-x initialization (called during module initialization).
823	*
824	* @returns VBox status code.
825	*/
826	VMMR0DECL(int) VMXR0GlobalInit(void)
827	{
828	/* Setup the main VM exit handlers. */
829	/** @todo I'm told lookup table for function pointers is a bad idea in terms
830	* of performance. Make these static for now and not do change anything
831	* at runtime so we can easily switch to a switch-case approach later. */
832	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
833	#ifdef DEBUG
834	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
835	Assert(s_apfnVMExitHandlers[i]);
836	#endif
837	return VINF_SUCCESS;
838	}
839
840
841	/**
842	* Does global VT-x termination (called during module termination).
843	*/
844	VMMR0DECL(void) VMXR0GlobalTerm()
845	{
846	/* Nothing to do currently. */
847	}
848
849
850	/**
851	* Sets up and activates VT-x on the current CPU.
852	*
853	* @returns VBox status code.
854	* @param pCpu Pointer to the global CPU info struct.
855	* @param pVM Pointer to the VM (can be NULL after a host resume
856	* operation).
857	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
858	* fEnabledByHost is true).
859	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
860	* @a fEnabledByHost is true).
861	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
862	* enable VT-x/AMD-V on the host.
863	*/
864	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
865	{
866	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
867	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
868
869	if (!fEnabledByHost)
870	{
871	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
872	if (RT_FAILURE(rc))
873	return rc;
874	}
875
876	/*
877	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
878	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
879	* each time while reusing a VPID after hitting the MaxASID limit once.
880	*/
881	if ( pVM
882	&& pVM->hm.s.vmx.fVpid
883	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
884	{
885	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
886	pCpu->fFlushAsidBeforeUse = false;
887	}
888	else
889	pCpu->fFlushAsidBeforeUse = true;
890
891	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
892	++pCpu->cTlbFlushes;
893
894	return VINF_SUCCESS;
895	}
896
897
898	/**
899	* Deactivates VT-x on the current CPU.
900	*
901	* @returns VBox status code.
902	* @param pCpu Pointer to the global CPU info struct.
903	* @param pvCpuPage Pointer to the VMXON region.
904	* @param HCPhysCpuPage Physical address of the VMXON region.
905	*/
906	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
907	{
908	NOREF(pCpu);
909	NOREF(pvCpuPage);
910	NOREF(HCPhysCpuPage);
911
912	hmR0VmxLeaveRootMode();
913	return VINF_SUCCESS;
914	}
915
916
917	/**
918	* Sets the permission bits for the specified MSR in the MSR bitmap.
919	*
920	* @param pVCpu Pointer to the VMCPU.
921	* @param ulMSR The MSR value.
922	* @param enmRead Whether reading this MSR causes a VM-exit.
923	* @param enmWrite Whether writing this MSR causes a VM-exit.
924	*/
925	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
926	{
927	unsigned ulBit;
928	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
929
930	/*
931	* Layout:
932	* 0x000 - 0x3ff - Low MSR read bits
933	* 0x400 - 0x7ff - High MSR read bits
934	* 0x800 - 0xbff - Low MSR write bits
935	* 0xc00 - 0xfff - High MSR write bits
936	*/
937	if (ulMsr <= 0x00001FFF)
938	{
939	/* Pentium-compatible MSRs */
940	ulBit = ulMsr;
941	}
942	else if ( ulMsr >= 0xC0000000
943	&& ulMsr <= 0xC0001FFF)
944	{
945	/* AMD Sixth Generation x86 Processor MSRs */
946	ulBit = (ulMsr - 0xC0000000);
947	pbMsrBitmap += 0x400;
948	}
949	else
950	{
951	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
952	return;
953	}
954
955	Assert(ulBit <= 0x1fff);
956	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
957	ASMBitSet(pbMsrBitmap, ulBit);
958	else
959	ASMBitClear(pbMsrBitmap, ulBit);
960
961	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
962	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
963	else
964	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
965	}
966
967
968	/**
969	* Flushes the TLB using EPT.
970	*
971	* @returns VBox status code.
972	* @param pVM Pointer to the VM.
973	* @param pVCpu Pointer to the VMCPU.
974	* @param enmFlush Type of flush.
975	*/
976	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
977	{
978	AssertPtr(pVM);
979	Assert(pVM->hm.s.fNestedPaging);
980
981	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
982
983	uint64_t descriptor[2];
984	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
985	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
986
987	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
988	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
989	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
990	}
991
992
993	/**
994	* Flushes the TLB using VPID.
995	*
996	* @returns VBox status code.
997	* @param pVM Pointer to the VM.
998	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
999	* enmFlush).
1000	* @param enmFlush Type of flush.
1001	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1002	* on @a enmFlush).
1003	*/
1004	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1005	{
1006	AssertPtr(pVM);
1007	Assert(pVM->hm.s.vmx.fVpid);
1008
1009	uint64_t descriptor[2];
1010	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1011	{
1012	descriptor[0] = 0;
1013	descriptor[1] = 0;
1014	}
1015	else
1016	{
1017	AssertPtr(pVCpu);
1018	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1019	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1020	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1021	descriptor[1] = GCPtr;
1022	}
1023
1024	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1025	AssertMsg(rc == VINF_SUCCESS,
1026	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1027	if ( RT_SUCCESS(rc)
1028	&& pVCpu)
1029	{
1030	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1031	}
1032	}
1033
1034
1035	/**
1036	* Invalidates a guest page by guest virtual address. Only relevant for
1037	* EPT/VPID, otherwise there is nothing really to invalidate.
1038	*
1039	* @returns VBox status code.
1040	* @param pVM Pointer to the VM.
1041	* @param pVCpu Pointer to the VMCPU.
1042	* @param GCVirt Guest virtual address of the page to invalidate.
1043	*/
1044	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1045	{
1046	AssertPtr(pVM);
1047	AssertPtr(pVCpu);
1048	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1049
1050	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1051	if (!fFlushPending)
1052	{
1053	/*
1054	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1055	* See @bugref{6043} and @bugref{6177}.
1056	*
1057	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1058	* function maybe called in a loop with individual addresses.
1059	*/
1060	if (pVM->hm.s.vmx.fVpid)
1061	{
1062	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1063	{
1064	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1065	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1066	}
1067	else
1068	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1069	}
1070	else if (pVM->hm.s.fNestedPaging)
1071	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1072	}
1073
1074	return VINF_SUCCESS;
1075	}
1076
1077
1078	/**
1079	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1080	* otherwise there is nothing really to invalidate.
1081	*
1082	* @returns VBox status code.
1083	* @param pVM Pointer to the VM.
1084	* @param pVCpu Pointer to the VMCPU.
1085	* @param GCPhys Guest physical address of the page to invalidate.
1086	*/
1087	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1088	{
1089	LogFlowFunc(("%RGp\n", GCPhys));
1090
1091	/*
1092	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1093	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1094	* This function might be called in a loop.
1095	*/
1096	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1097	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1098	return VINF_SUCCESS;
1099	}
1100
1101
1102	/**
1103	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1104	* case where neither EPT nor VPID is supported by the CPU.
1105	*
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU.
1108	*
1109	* @remarks Called with interrupts disabled.
1110	*/
1111	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1112	{
1113	NOREF(pVM);
1114	AssertPtr(pVCpu);
1115	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1116	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1117
1118	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1119	AssertPtr(pCpu);
1120
1121	pVCpu->hm.s.TlbShootdown.cPages = 0;
1122	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1123	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1124	pVCpu->hm.s.fForceTLBFlush = false;
1125	return;
1126	}
1127
1128
1129	/**
1130	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	* @remarks All references to "ASID" in this function pertains to "VPID" in
1135	* Intel's nomenclature. The reason is, to avoid confusion in compare
1136	* statements since the host-CPU copies are named "ASID".
1137	*
1138	* @remarks Called with interrupts disabled.
1139	* @todo Statistics.
1140	*/
1141	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1142	{
1143	AssertPtr(pVM);
1144	AssertPtr(pVCpu);
1145	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1146	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1147	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1148
1149	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1150	AssertPtr(pCpu);
1151
1152	/*
1153	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1154	* This can happen both for start & resume due to long jumps back to ring-3.
1155	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1156	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1157	*/
1158	bool fNewASID = false;
1159	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1160	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1161	{
1162	pVCpu->hm.s.fForceTLBFlush = true;
1163	fNewASID = true;
1164	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1165	}
1166
1167	/*
1168	* Check for explicit TLB shootdowns.
1169	*/
1170	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1171	{
1172	pVCpu->hm.s.fForceTLBFlush = true;
1173	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1174	}
1175
1176	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1177	if (pVCpu->hm.s.fForceTLBFlush)
1178	{
1179	if (fNewASID)
1180	{
1181	++pCpu->uCurrentAsid;
1182	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1183	{
1184	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1185	pCpu->cTlbFlushes++;
1186	pCpu->fFlushAsidBeforeUse = true;
1187	}
1188
1189	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1190	if (pCpu->fFlushAsidBeforeUse)
1191	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1192	}
1193	else
1194	{
1195	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1196	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1197	else
1198	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1199	}
1200
1201	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1202	pVCpu->hm.s.fForceTLBFlush = false;
1203	}
1204	else
1205	{
1206	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1207	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1208	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1209	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1210
1211	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1212	* not be executed. See hmQueueInvlPage() where it is commented
1213	* out. Support individual entry flushing someday. */
1214	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1215	{
1216	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1217
1218	/*
1219	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1220	* as supported by the CPU.
1221	*/
1222	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223	{
1224	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1225	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1226	}
1227	else
1228	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1229	}
1230	else
1231	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1232	}
1233	pVCpu->hm.s.TlbShootdown.cPages = 0;
1234	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1235
1236	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1237	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1238	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1239	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1240	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1241	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1242
1243	/* Update VMCS with the VPID. */
1244	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1245	AssertRC(rc);
1246	}
1247
1248
1249	/**
1250	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1251	*
1252	* @returns VBox status code.
1253	* @param pVM Pointer to the VM.
1254	* @param pVCpu Pointer to the VMCPU.
1255	*
1256	* @remarks Called with interrupts disabled.
1257	* @todo Statistics.
1258	*/
1259	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1260	{
1261	AssertPtr(pVM);
1262	AssertPtr(pVCpu);
1263	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1264	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1265
1266	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1267	AssertPtr(pCpu);
1268
1269	/*
1270	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1271	* This can happen both for start & resume due to long jumps back to ring-3.
1272	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1273	*/
1274	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1275	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1276	{
1277	pVCpu->hm.s.fForceTLBFlush = true;
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1279	}
1280
1281	/* Check for explicit TLB shootdown flushes. */
1282	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1283	{
1284	pVCpu->hm.s.fForceTLBFlush = true;
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1286	}
1287
1288	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1289	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1290
1291	if (pVCpu->hm.s.fForceTLBFlush)
1292	{
1293	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1294	pVCpu->hm.s.fForceTLBFlush = false;
1295	}
1296	else
1297	{
1298	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1299	* not be executed. See hmQueueInvlPage() where it is commented
1300	* out. Support individual entry flushing someday. */
1301	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1302	{
1303	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1305	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1306	}
1307	else
1308	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1309	}
1310
1311	pVCpu->hm.s.TlbShootdown.cPages= 0;
1312	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1313	}
1314
1315
1316	/**
1317	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1318	*
1319	* @returns VBox status code.
1320	* @param pVM Pointer to the VM.
1321	* @param pVCpu Pointer to the VMCPU.
1322	*
1323	* @remarks Called with interrupts disabled.
1324	* @todo Statistics.
1325	*/
1326	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1327	{
1328	AssertPtr(pVM);
1329	AssertPtr(pVCpu);
1330	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1331	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1332
1333	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1334
1335	/*
1336	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1337	* This can happen both for start & resume due to long jumps back to ring-3.
1338	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1339	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1340	*/
1341	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1342	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1343	{
1344	pVCpu->hm.s.fForceTLBFlush = true;
1345	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1346	}
1347
1348	/* Check for explicit TLB shootdown flushes. */
1349	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1350	{
1351	/*
1352	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1353	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1354	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1355	*/
1356	pVCpu->hm.s.fForceTLBFlush = true;
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1358	}
1359
1360	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1361	if (pVCpu->hm.s.fForceTLBFlush)
1362	{
1363	++pCpu->uCurrentAsid;
1364	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1365	{
1366	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1367	pCpu->fFlushAsidBeforeUse = true;
1368	pCpu->cTlbFlushes++;
1369	}
1370
1371	pVCpu->hm.s.fForceTLBFlush = false;
1372	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1373	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1374	if (pCpu->fFlushAsidBeforeUse)
1375	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1376	}
1377	else
1378	{
1379	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1380	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1381	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1382	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1383
1384	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1385	* not be executed. See hmQueueInvlPage() where it is commented
1386	* out. Support individual entry flushing someday. */
1387	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1388	{
1389	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1390	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1391	{
1392	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1393	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1394	}
1395	else
1396	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1397	}
1398	else
1399	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1400	}
1401
1402	pVCpu->hm.s.TlbShootdown.cPages = 0;
1403	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1404
1405	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1406	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1407	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1408	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1409	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1410	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1411
1412	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1413	AssertRC(rc);
1414	}
1415
1416
1417	/**
1418	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1419	* TLB entries from the host TLB before VM-entry.
1420	*
1421	* @returns VBox status code.
1422	* @param pVM Pointer to the VM.
1423	*/
1424	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1425	{
1426	/*
1427	* Determine optimal flush type for nested paging.
1428	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1429	* guest execution (see hmR3InitFinalizeR0()).
1430	*/
1431	if (pVM->hm.s.fNestedPaging)
1432	{
1433	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1434	{
1435	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1436	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1437	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1438	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1439	else
1440	{
1441	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1442	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1443	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1444	}
1445
1446	/* Make sure the write-back cacheable memory type for EPT is supported. */
1447	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1448	{
1449	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1450	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1451	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1452	}
1453	}
1454	else
1455	{
1456	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1457	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1458	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1459	}
1460	}
1461
1462	/*
1463	* Determine optimal flush type for VPID.
1464	*/
1465	if (pVM->hm.s.vmx.fVpid)
1466	{
1467	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1468	{
1469	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1470	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1471	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1472	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1473	else
1474	{
1475	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1476	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1477	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1478	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1479	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1480	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1481	pVM->hm.s.vmx.fVpid = false;
1482	}
1483	}
1484	else
1485	{
1486	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1487	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1488	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1489	pVM->hm.s.vmx.fVpid = false;
1490	}
1491	}
1492
1493	/*
1494	* Setup the handler for flushing tagged-TLBs.
1495	*/
1496	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1497	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1498	else if (pVM->hm.s.fNestedPaging)
1499	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1500	else if (pVM->hm.s.vmx.fVpid)
1501	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1502	else
1503	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1504	return VINF_SUCCESS;
1505	}
1506
1507
1508	/**
1509	* Sets up pin-based VM-execution controls in the VMCS.
1510	*
1511	* @returns VBox status code.
1512	* @param pVM Pointer to the VM.
1513	* @param pVCpu Pointer to the VMCPU.
1514	*/
1515	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1516	{
1517	AssertPtr(pVM);
1518	AssertPtr(pVCpu);
1519
1520	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1521	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1522
1523	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1524	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1525	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1526
1527	/* Enable the VMX preemption timer. */
1528	if (pVM->hm.s.vmx.fUsePreemptTimer)
1529	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1530
1531	if ((val & zap) != val)
1532	{
1533	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1534	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1535	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1536	}
1537
1538	val &= zap;
1539	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1540	AssertRCReturn(rc, rc);
1541
1542	/* Update VCPU with the currently set pin-based VM-execution controls. */
1543	pVCpu->hm.s.vmx.u32PinCtls = val;
1544	return rc;
1545	}
1546
1547
1548	/**
1549	* Sets up processor-based VM-execution controls in the VMCS.
1550	*
1551	* @returns VBox status code.
1552	* @param pVM Pointer to the VM.
1553	* @param pVMCPU Pointer to the VMCPU.
1554	*/
1555	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1556	{
1557	AssertPtr(pVM);
1558	AssertPtr(pVCpu);
1559
1560	int rc = VERR_INTERNAL_ERROR_5;
1561	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1562	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1563
1564	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1565	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1566	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1567	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1568	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1569	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1570	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1571
1572	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1573	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1574	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1575	{
1576	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1577	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1578	}
1579
1580	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1581	if (!pVM->hm.s.fNestedPaging)
1582	{
1583	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1584	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1585	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1586	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1587	}
1588
1589	/* Use TPR shadowing if supported by the CPU. */
1590	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1591	{
1592	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1593	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1594	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1595	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1596
1597	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1598	/* CR8 writes causes a VM-exit based on TPR threshold. */
1599	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1600	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1601	}
1602	else
1603	{
1604	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1605	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1606	}
1607
1608	/* Use MSR-bitmaps if supported by the CPU. */
1609	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1610	{
1611	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1612
1613	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1614	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1615	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1616	AssertRCReturn(rc, rc);
1617
1618	/*
1619	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1620	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1621	*/
1622	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1623	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1624	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1625	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1626	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1627	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1628	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1629	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1630	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1631	}
1632
1633	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1634	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1635	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1636
1637	if ((val & zap) != val)
1638	{
1639	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1640	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1641	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1642	}
1643
1644	val &= zap;
1645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1646	AssertRCReturn(rc, rc);
1647
1648	/* Update VCPU with the currently set processor-based VM-execution controls. */
1649	pVCpu->hm.s.vmx.u32ProcCtls = val;
1650
1651	/*
1652	* Secondary processor-based VM-execution controls.
1653	*/
1654	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655	{
1656	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1657	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1658
1659	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1660
1661	if (pVM->hm.s.fNestedPaging)
1662	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1663	else
1664	{
1665	/*
1666	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1667	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1668	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1669	*/
1670	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1671	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1672	}
1673
1674	if (pVM->hm.s.vmx.fVpid)
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1676
1677	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1678	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1679
1680	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1681	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1682	* done dynamically. */
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1684	{
1685	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1686	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1687	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1688	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1689	AssertRCReturn(rc, rc);
1690	}
1691
1692	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1693	{
1694	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1696	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1702	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	val &= zap;
1707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1708	AssertRCReturn(rc, rc);
1709
1710	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1711	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1712	}
1713
1714	return VINF_SUCCESS;
1715	}
1716
1717
1718	/**
1719	* Sets up miscellaneous (everything other than Pin & Processor-based
1720	* VM-execution) control fields in the VMCS.
1721	*
1722	* @returns VBox status code.
1723	* @param pVM Pointer to the VM.
1724	* @param pVCpu Pointer to the VMCPU.
1725	*/
1726	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1727	{
1728	AssertPtr(pVM);
1729	AssertPtr(pVCpu);
1730
1731	int rc = VERR_GENERAL_FAILURE;
1732
1733	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1735
1736	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1737
1738	/*
1739	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1740	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1741	* We thus use the exception bitmap to control it rather than use both.
1742	*/
1743	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1744	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1745
1746	/** @todo Explore possibility of using IO-bitmaps. */
1747	/* All IO & IOIO instructions cause VM-exits. */
1748	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1749	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1750
1751	/* Setup MSR autoloading/autostoring. */
1752	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1753	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1754	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1755	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1756	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1757	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1758
1759	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1762	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1763
1764	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1765	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1766
1767	/* Setup debug controls */
1768	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1769	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1770	AssertRCReturn(rc, rc);
1771	return rc;
1772	}
1773
1774
1775	/**
1776	* Sets up the initial exception bitmap in the VMCS based on static conditions
1777	* (i.e. conditions that cannot ever change at runtime).
1778	*
1779	* @returns VBox status code.
1780	* @param pVM Pointer to the VM.
1781	* @param pVCpu Pointer to the VMCPU.
1782	*/
1783	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1784	{
1785	AssertPtr(pVM);
1786	AssertPtr(pVCpu);
1787
1788	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1789
1790	uint32_t u32XcptBitmap = 0;
1791
1792	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1793	if (!pVM->hm.s.fNestedPaging)
1794	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1795
1796	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1797	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1798	AssertRCReturn(rc, rc);
1799	return rc;
1800	}
1801
1802
1803	/**
1804	* Sets up the initial guest-state mask. The guest-state mask is consulted
1805	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1806	* for the nested virtualization case (as it would cause a VM-exit).
1807	*
1808	* @param pVCpu Pointer to the VMCPU.
1809	*/
1810	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1811	{
1812	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1813	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1814	return VINF_SUCCESS;
1815	}
1816
1817
1818	/**
1819	* Does per-VM VT-x initialization.
1820	*
1821	* @returns VBox status code.
1822	* @param pVM Pointer to the VM.
1823	*/
1824	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1825	{
1826	LogFlowFunc(("pVM=%p\n", pVM));
1827
1828	int rc = hmR0VmxStructsAlloc(pVM);
1829	if (RT_FAILURE(rc))
1830	{
1831	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1832	return rc;
1833	}
1834
1835	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1836	{
1837	PVMCPU pVCpu = &pVM->aCpus[i];
1838
1839	/* Current guest paging mode. */
1840	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1841	}
1842
1843	return VINF_SUCCESS;
1844	}
1845
1846
1847	/**
1848	* Does per-VM VT-x termination.
1849	*
1850	* @returns VBox status code.
1851	* @param pVM Pointer to the VM.
1852	*/
1853	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1854	{
1855	LogFlowFunc(("pVM=%p\n", pVM));
1856
1857	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1858	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1859	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1860	#endif
1861	hmR0VmxStructsFree(pVM);
1862	return VINF_SUCCESS;
1863	}
1864
1865
1866	/**
1867	* Sets up the VM for execution under VT-x.
1868	* This function is only called once per-VM during initalization.
1869	*
1870	* @returns VBox status code.
1871	* @param pVM Pointer to the VM.
1872	*/
1873	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1874	{
1875	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1876	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1877
1878	LogFlowFunc(("pVM=%p\n", pVM));
1879
1880	/*
1881	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1882	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1883	*/
1884	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1885	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1886	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1887	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1888	{
1889	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1890	return VERR_INTERNAL_ERROR;
1891	}
1892
1893	/* Initialize these always, see hmR3InitFinalizeR0().*/
1894	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1895	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1896
1897	/* Setup the tagged-TLB flush handlers. */
1898	int rc = hmR0VmxSetupTaggedTlb(pVM);
1899	if (RT_FAILURE(rc))
1900	{
1901	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1902	return rc;
1903	}
1904
1905	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1906	{
1907	PVMCPU pVCpu = &pVM->aCpus[i];
1908	AssertPtr(pVCpu);
1909	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1910
1911	/* Set revision dword at the beginning of the VMCS structure. */
1912	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1913
1914	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1915	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1916	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1917	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1918
1919	/* Load this VMCS as the current VMCS. */
1920	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1921	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1922	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1923
1924	/* Setup the pin-based VM-execution controls. */
1925	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1926	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1927	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1928
1929	/* Setup the processor-based VM-execution controls. */
1930	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1931	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1932	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1933
1934	/* Setup the rest (miscellaneous) VM-execution controls. */
1935	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1936	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1937	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1938
1939	/* Setup the initial exception bitmap. */
1940	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1941	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1942	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1943
1944	/* Setup the initial guest-state mask. */
1945	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1946	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1947	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1948
1949	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1950	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1951	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1952	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1953	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1954	#endif
1955
1956	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1957	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	/* Update the last error record for this VCPU. */
1962	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1963	}
1964
1965	return VINF_SUCCESS;
1966	}
1967
1968
1969	/**
1970	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1971	* the VMCS.
1972	*
1973	* @returns VBox status code.
1974	* @param pVM Pointer to the VM.
1975	* @param pVCpu Pointer to the VMCPU.
1976	*/
1977	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1978	{
1979	RTCCUINTREG uReg = ASMGetCR0();
1980	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1981	AssertRCReturn(rc, rc);
1982
1983	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1984	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1985	if (VMX_IS_64BIT_HOST_MODE())
1986	{
1987	uint64_t uReg = hmR0Get64bitCR3();
1988	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
1989	}
1990	else
1991	#endif
1992	{
1993	uReg = ASMGetCR3();
1994	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
1995	}
1996	AssertRCReturn(rc, rc);
1997
1998	uReg = ASMGetCR4();
1999	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2000	AssertRCReturn(rc, rc);
2001	return rc;
2002	}
2003
2004
2005	/**
2006	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2007	* the host-state area in the VMCS.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	* @param pVCpu Pointer to the VMCPU.
2012	*/
2013	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2014	{
2015	int rc = VERR_INTERNAL_ERROR_5;
2016	RTSEL uSelCS = 0;
2017	RTSEL uSelSS = 0;
2018	RTSEL uSelDS = 0;
2019	RTSEL uSelES = 0;
2020	RTSEL uSelFS = 0;
2021	RTSEL uSelGS = 0;
2022	RTSEL uSelTR = 0;
2023
2024	/*
2025	* Host Selector registers.
2026	*/
2027	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2028	if (VMX_IS_64BIT_HOST_MODE())
2029	{
2030	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2031	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2032	}
2033	else
2034	{
2035	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2036	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2037	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2038	}
2039	#else
2040	uSelCS = ASMGetCS();
2041	uSelSS = ASMGetSS();
2042	#endif
2043
2044	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2045	/** @todo Verify if we have any platform that actually run with DS or ES with
2046	* RPL != 0 in kernel space. */
2047	uSelDS = 0;
2048	uSelES = 0;
2049	uSelFS = 0;
2050	uSelGS = 0;
2051	uSelTR = ASMGetTR();
2052
2053	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2054	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2055	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2056	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2057	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2058	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2059	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2060	Assert(uSelCS != 0);
2061	Assert(uSelTR != 0);
2062
2063	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2064	#if 0
2065	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2066	Assert(uSelSS != 0);
2067	#endif
2068
2069	/* Write these host selector fields into the host-state area in the VMCS. */
2070	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2071	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2072	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2073	#if 0
2074	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2075	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2076	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2077	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2078	#endif
2079	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2080	AssertRCReturn(rc, rc);
2081
2082	/*
2083	* Host GDTR and IDTR.
2084	*/
2085	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2086	* be safe to -not- save and restore GDTR and IDTR in the assembly
2087	* code and just do it here and don't care if the limits are zapped on
2088	* VM-exit. */
2089	RTGDTR Gdtr;
2090	RT_ZERO(Gdtr);
2091	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092	if (VMX_IS_64BIT_HOST_MODE())
2093	{
2094	X86XDTR64 Gtr64;
2095	X86XDTR64 Idtr64;
2096	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2097	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2098	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2099	Gdtr.cbGdt = Gdtr64.cb;
2100	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2101	}
2102	else
2103	#endif
2104	{
2105	RTIDTR Idtr;
2106	ASMGetGDTR(&Gdtr);
2107	ASMGetIDTR(&Idtr);
2108	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2109	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2110	}
2111	AssertRCReturn(rc, rc);
2112
2113	/*
2114	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2115	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2116	*/
2117	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2118	{
2119	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2120	return VERR_VMX_INVALID_HOST_STATE;
2121	}
2122
2123	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2124	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2125	if (VMX_IS_64BIT_HOST_MODE())
2126	{
2127	/* We need the 64-bit TR base for hybrid darwin. */
2128	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2129	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2130	}
2131	else
2132	#endif
2133	{
2134	uintptr_t uTRBase;
2135	#if HC_ARCH_BITS == 64
2136	uTRBase = X86DESC64_BASE(pDesc);
2137	#else
2138	uTRBase = X86DESC_BASE(pDesc);
2139	#endif
2140	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2141	}
2142	AssertRCReturn(rc, rc);
2143
2144	/*
2145	* Host FS base and GS base.
2146	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2147	* would take care of the bases. In 64-bit, the MSRs come into play.
2148	*/
2149	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2150	if (VMX_IS_64BIT_HOST_MODE())
2151	{
2152	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2153	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2154	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2155	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2156	AssertRCReturn(rc, rc);
2157	}
2158	#endif
2159	return rc;
2160	}
2161
2162
2163	/**
2164	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2165	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2166	* the host after every successful VM exit.
2167	*
2168	* @returns VBox status code.
2169	* @param pVM Pointer to the VM.
2170	* @param pVCpu Pointer to the VMCPU.
2171	*/
2172	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2173	{
2174	AssertPtr(pVCpu);
2175	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2176
2177	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2178	unsigned idxHostMsr = 0;
2179	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2180
2181	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2182	{
2183	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2184	pHostMsr->u32Reserved = 0;
2185	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2186	if (CPUMIsGuestInLongMode(pVCpu))
2187	{
2188	/* Must match the EFER value in our 64 bits switcher. */
2189	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2190	}
2191	else
2192	#endif
2193	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2194	pHostMsr++; idxHostMsr++;
2195	}
2196
2197	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2198	if (VMX_IS_64BIT_HOST_MODE())
2199	{
2200	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2201	pHostMsr->u32Reserved = 0;
2202	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2203	pHostMsr++; idxHostMsr++;
2204	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2205	pHostMsr->u32Reserved = 0;
2206	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2207	pHostMsr++; idxHostMsr++;
2208	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2209	pHostMsr->u32Reserved = 0;
2210	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2211	pHostMsr++; idxHostMsr++;
2212	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2213	#if 0
2214	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2215	pMsr->u32Reserved = 0;
2216	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2217	pHostMsr++; idxHostMsr++;
2218	#endif
2219	}
2220	#endif
2221
2222	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2223	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2224	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2225
2226	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2227	AssertRCReturn(rc, rc);
2228
2229	/*
2230	* Host Sysenter MSRs.
2231	*/
2232	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2233	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2234	if (VMX_IS_64BIT_HOST_MODE())
2235	{
2236	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2237	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2238	}
2239	else
2240	{
2241	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2242	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2243	}
2244	#elif HC_ARCH_BITS == 32
2245	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2246	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2247	#else
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2249	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2250	#endif
2251	AssertRCReturn(rc, rc);
2252
2253	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2254	* hmR0VmxSetupExitCtls() !! */
2255	return rc;
2256	}
2257
2258
2259	/**
2260	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2261	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2262	* controls".
2263	*
2264	* @returns VBox status code.
2265	* @param pVM Pointer to the VM.
2266	* @param pVCpu Pointer to the VMCPU.
2267	* @param pCtx Pointer to the guest-CPU context.
2268	*
2269	* @remarks No-long-jump zone!!!
2270	*/
2271	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2272	{
2273	int rc = VINF_SUCCESS;
2274	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2275	{
2276	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2277	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2278
2279	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2280	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2281
2282	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2283	if (CPUMIsGuestInLongModeEx(pCtx))
2284	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2285	else
2286	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2287
2288	/*
2289	* The following should not be set (since we're not in SMM mode):
2290	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2291	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2292	*/
2293
2294	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2295	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2296	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2297
2298	if ((val & zap) != val)
2299	{
2300	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2301	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2302	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2303	}
2304
2305	val &= zap;
2306	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2307	AssertRCReturn(rc, rc);
2308
2309	/* Update VCPU with the currently set VM-exit controls. */
2310	pVCpu->hm.s.vmx.u32EntryCtls = val;
2311	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2312	}
2313	return rc;
2314	}
2315
2316
2317	/**
2318	* Sets up the VM-exit controls in the VMCS.
2319	*
2320	* @returns VBox status code.
2321	* @param pVM Pointer to the VM.
2322	* @param pVCpu Pointer to the VMCPU.
2323	*/
2324	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2325	{
2326	int rc = VINF_SUCCESS;
2327	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2328	{
2329	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2330	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2331
2332	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2333	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2334
2335	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2336	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2337	if (VMX_IS_64BIT_HOST_MODE())
2338	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2339	else
2340	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2341	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2342	if (CPUMIsGuestInLongModeEx(pCtx))
2343	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2344	else
2345	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2346	#endif
2347
2348	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2349	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2350
2351	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2352	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2353	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2354	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2355	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2356
2357	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2358	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2359
2360	if ((val & zap) != val)
2361	{
2362	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2363	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2364	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2365	}
2366
2367	val &= zap;
2368	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2369
2370	/* Update VCPU with the currently set VM-exit controls. */
2371	pVCpu->hm.s.vmx.u32ExitCtls = val;
2372	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2373	}
2374	return rc;
2375	}
2376
2377
2378	/**
2379	* Loads the guest APIC and related state.
2380	*
2381	* @returns VBox status code.
2382	* @param pVM Pointer to the VM.
2383	* @param pVCpu Pointer to the VMCPU.
2384	* @param pCtx Pointer to the guest-CPU context.
2385	*/
2386	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2387	{
2388	int rc = VINF_SUCCESS;
2389	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2390	{
2391	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2392	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2393	{
2394	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395
2396	bool fPendingIntr = false;
2397	uint8_t u8GuestTpr = 0;
2398	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2399	AssertRCReturn(rc, rc);
2400
2401	/*
2402	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2403	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2404	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2405	* the interrupt when we VM-exit for other reasons.
2406	*/
2407	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2408	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2409	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2410	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2411
2412	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2413	AssertRCReturn(rc, rc);
2414
2415	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2416	if (pVM->hm.s.fTPRPatchingActive)
2417	{
2418	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2419	pCtx->msrLSTAR = u8GuestTpr;
2420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2421	{
2422	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2423	if (fPendingIntr)
2424	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2425	else
2426	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427	}
2428	}
2429	}
2430
2431	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2432	}
2433	return rc;
2434	}
2435
2436
2437	/**
2438	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2439	* into the guest-state area in the VMCS.
2440	*
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2444	* out-of-sync. Make sure to update the required fields
2445	* before using them.
2446	*
2447	* @remarks No-long-jump zone!!!
2448	*/
2449	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2450	{
2451	/*
2452	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2453	* inhibit interrupts or clear any existing interrupt-inhibition.
2454	*/
2455	uint32_t uIntrState = 0;
2456	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2457	{
2458	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2459	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2460	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2461	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2462	{
2463	/*
2464	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2465	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2466	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2467	*/
2468	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2469	}
2470	else if (pMixedCtx->eflags.Bits.u1IF)
2471	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2472	else
2473	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2474	}
2475
2476	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2477	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2478	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2479	AssertRC(rc);
2480	}
2481
2482
2483	/**
2484	* Loads the guest's RIP into the guest-state area in the VMCS.
2485	*
2486	* @returns VBox status code.
2487	* @param pVM Pointer to the VM.
2488	* @param pVCpu Pointer to the VMCPU.
2489	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2490	* out-of-sync. Make sure to update the required fields
2491	* before using them.
2492	*
2493	* @remarks No-long-jump zone!!!
2494	*/
2495	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2496	{
2497	int rc = VINF_SUCCESS;
2498	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2499	{
2500	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2501	AssertRCReturn(rc, rc);
2502	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2503	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2504	}
2505	return rc;
2506	}
2507
2508
2509	/**
2510	* Loads the guest's RSP into the guest-state area in the VMCS.
2511	*
2512	* @returns VBox status code.
2513	* @param pVM Pointer to the VM.
2514	* @param pVCpu Pointer to the VMCPU.
2515	* @param pCtx Pointer to the guest-CPU context.
2516	*
2517	* @remarks No-long-jump zone!!!
2518	*/
2519	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2520	{
2521	int rc = VINF_SUCCESS;
2522	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2523	{
2524	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2525	AssertRCReturn(rc, rc);
2526	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2527	}
2528	return rc;
2529	}
2530
2531
2532	/**
2533	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2534	*
2535	* @returns VBox status code.
2536	* @param pVM Pointer to the VM.
2537	* @param pVCpu Pointer to the VMCPU.
2538	* @param pCtx Pointer to the guest-CPU context.
2539	*
2540	* @remarks No-long-jump zone!!!
2541	*/
2542	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2543	{
2544	int rc = VINF_SUCCESS;
2545	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2546	{
2547	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2548	Let us assert it as such and use native-width VMWRITE. */
2549	X86RFLAGS uRFlags = pCtx->rflags;
2550	Assert(uRFlags.u64 >> 32 == 0);
2551	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2552	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2553
2554	/*
2555	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2556	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2557	*/
2558	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2559	{
2560	Assert(pVM->hm.s.vmx.pRealModeTSS);
2561	Assert(PDMVmmDevHeapIsEnabled(pVM));
2562	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2563	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2564	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2565	}
2566
2567	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2568	AssertRCReturn(rc, rc);
2569
2570	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2571	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2572	}
2573	return rc;
2574	}
2575
2576
2577	/**
2578	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2579	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2580	* assembly code.
2581	*
2582	* @returns VBox status code.
2583	* @param pVM Pointer to the VM.
2584	* @param pVCpu Pointer to the VMCPU.
2585	* @param pCtx Pointer to the guest-CPU context.
2586	*
2587	* @remarks No-long-jump zone!!!
2588	*/
2589	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2590	{
2591	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2592	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2593	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2594	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2595	AssertRCReturn(rc, rc);
2596	return rc;
2597	}
2598
2599
2600	/**
2601	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2602	* in the VMCS.
2603	*
2604	* @returns VBox status code.
2605	* @param pVM Pointer to the VM.
2606	* @param pVCpu Pointer to the VMCPU.
2607	* @param pCtx Pointer to the guest-CPU context.
2608	*
2609	* @remarks No-long-jump zone!!!
2610	*/
2611	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2612	{
2613	int rc = VINF_SUCCESS;
2614
2615	/*
2616	* Guest CR0.
2617	* Guest FPU.
2618	*/
2619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2620	{
2621	uint64_t u64GuestCR0 = pCtx->cr0;
2622
2623	/* The guest's view (read access) of its CR0 is unblemished. */
2624	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2625	AssertRCReturn(rc, rc);
2626	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2627
2628	/* Setup VT-x's view of the guest CR0. */
2629	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2630	if (pVM->hm.s.fNestedPaging)
2631	{
2632	if (CPUMIsGuestPagingEnabledEx(pCtx))
2633	{
2634	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2635	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2636	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2637	}
2638	else
2639	{
2640	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2641	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2643	}
2644
2645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2646	AssertRCReturn(rc, rc);
2647	}
2648	else
2649	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2650
2651	/*
2652	* Guest FPU bits.
2653	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2654	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2655	*/
2656	u64GuestCR0 \|= X86_CR0_NE;
2657	bool fInterceptNM = false;
2658	if (CPUMIsGuestFPUStateActive(pVCpu))
2659	{
2660	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2661	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2662	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2663	}
2664	else
2665	{
2666	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2667	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2668	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2669	}
2670
2671	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2672	bool fInterceptMF = false;
2673	if (!(pCtx->cr0 & X86_CR0_NE))
2674	fInterceptMF = true;
2675
2676	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2677	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2678	{
2679	Assert(PDMVmmDevHeapIsEnabled(pVM));
2680	Assert(pVM->hm.s.vmx.pRealModeTSS);
2681	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2682	fInterceptNM = true;
2683	fInterceptMF = true;
2684	}
2685	else
2686	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2687
2688	if (fInterceptNM)
2689	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2690	else
2691	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2692
2693	if (fInterceptMF)
2694	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2695	else
2696	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2697
2698	/* Additional intercepts for debugging, define these yourself explicitly. */
2699	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2700	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2701	\| RT_BIT(X86_XCPT_DB)
2702	\| RT_BIT(X86_XCPT_DE)
2703	\| RT_BIT(X86_XCPT_NM)
2704	\| RT_BIT(X86_XCPT_UD)
2705	\| RT_BIT(X86_XCPT_NP)
2706	\| RT_BIT(X86_XCPT_SS)
2707	\| RT_BIT(X86_XCPT_GP)
2708	\| RT_BIT(X86_XCPT_PF)
2709	\| RT_BIT(X86_XCPT_MF);
2710	#elif defined(VBOX_ALWAYS_TRAP_PF)
2711	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2712	#endif
2713
2714	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2715
2716	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2717	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2718	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2719	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2720	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2721	else
2722	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2723
2724	u64GuestCR0 \|= uSetCR0;
2725	u64GuestCR0 &= uZapCR0;
2726	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2727
2728	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2729	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2730	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2731	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2732
2733	/*
2734	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2735	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2736	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2737	*/
2738	uint64_t u64CR0Mask = 0;
2739	u64CR0Mask = X86_CR0_PE
2740	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2741	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2742	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2743	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2744	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2745
2746	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2747	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2748	u64CR0Mask &= ~X86_CR0_PE;
2749
2750	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2751	if (fInterceptNM)
2752	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2753	else
2754	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2755
2756	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2757	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2758	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2759	AssertRCReturn(rc, rc);
2760
2761	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2762	}
2763
2764	/*
2765	* Guest CR2.
2766	* It's always loaded in the assembler code. Nothing to do here.
2767	*/
2768
2769	/*
2770	* Guest CR3.
2771	*/
2772	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2773	{
2774	uint64_t u64GuestCR3 = 0;
2775	if (pVM->hm.s.fNestedPaging)
2776	{
2777	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2778
2779	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2780	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2781	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2782	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2783
2784	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2785	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2786	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2787
2788	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2789	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2790	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2791	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2792
2793	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2794	AssertRCReturn(rc, rc);
2795	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2796
2797	if ( CPUMIsGuestPagingEnabledEx(pCtx)
2798	\|\| pVM->hm.s.vmx.fUnrestrictedGuest)
2799	{
2800	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2801	if (CPUMIsGuestInPAEModeEx(pCtx))
2802	{
2803	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2804	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2805	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2806	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2807	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2808	AssertRCReturn(rc, rc);
2809	}
2810
2811	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2812	have Unrestricted Execution to handle the guest when it's not using paging. */
2813	u64GuestCR3 = pCtx->cr3;
2814	}
2815	else
2816	{
2817	/*
2818	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2819	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2820	* EPT takes care of translating it to host-physical addresses.
2821	*/
2822	RTGCPHYS GCPhys;
2823	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2824	Assert(PDMVmmDevHeapIsEnabled(pVM));
2825
2826	/* We obtain it here every time as the guest could have relocated this PCI region. */
2827	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2828	AssertRCReturn(rc, rc);
2829
2830	u64GuestCR3 = GCPhys;
2831	}
2832	}
2833	else
2834	{
2835	/* Non-nested paging case, just use the hypervisor's CR3. */
2836	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2837	}
2838
2839	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2840	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2841	AssertRCReturn(rc, rc);
2842
2843	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2844	}
2845
2846	/*
2847	* Guest CR4.
2848	*/
2849	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2850	{
2851	uint64_t u64GuestCR4 = pCtx->cr4;
2852
2853	/* The guest's view of its CR4 is unblemished. */
2854	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2855	AssertRCReturn(rc, rc);
2856	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2857
2858	/* Setup VT-x's view of the guest CR4. */
2859	/*
2860	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2861	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2862	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2863	*/
2864	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2865	{
2866	Assert(pVM->hm.s.vmx.pRealModeTSS);
2867	Assert(PDMVmmDevHeapIsEnabled(pVM));
2868	u64GuestCR4 &= ~X86_CR4_VME;
2869	}
2870
2871	if (pVM->hm.s.fNestedPaging)
2872	{
2873	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2874	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2875	{
2876	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2877	u64GuestCR4 \|= X86_CR4_PSE;
2878	/* Our identity mapping is a 32 bits page directory. */
2879	u64GuestCR4 &= ~X86_CR4_PAE;
2880	}
2881	/* else use guest CR4.*/
2882	}
2883	else
2884	{
2885	/*
2886	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2887	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2888	*/
2889	switch (pVCpu->hm.s.enmShadowMode)
2890	{
2891	case PGMMODE_REAL: /* Real-mode. */
2892	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2893	case PGMMODE_32_BIT: /* 32-bit paging. */
2894	{
2895	u64GuestCR4 &= ~X86_CR4_PAE;
2896	break;
2897	}
2898
2899	case PGMMODE_PAE: /* PAE paging. */
2900	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2901	{
2902	u64GuestCR4 \|= X86_CR4_PAE;
2903	break;
2904	}
2905
2906	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2907	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2908	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2909	break;
2910	#endif
2911	default:
2912	AssertFailed();
2913	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2914	}
2915	}
2916
2917	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2918	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2919	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2920	u64GuestCR4 \|= uSetCR4;
2921	u64GuestCR4 &= uZapCR4;
2922
2923	/* Write VT-x's view of the guest CR4 into the VMCS. */
2924	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2925	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2926
2927	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2928	uint64_t u64CR4Mask = 0;
2929	u64CR4Mask = X86_CR4_VME
2930	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2931	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2932	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2933	\| X86_CR4_VMXE;
2934	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2935	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2936	AssertRCReturn(rc, rc);
2937
2938	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2939	}
2940	return rc;
2941	}
2942
2943
2944	/**
2945	* Loads the guest debug registers into the guest-state area in the VMCS.
2946	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2947	*
2948	* @returns VBox status code.
2949	* @param pVM Pointer to the VM.
2950	* @param pVCpu Pointer to the VMCPU.
2951	* @param pCtx Pointer to the guest-CPU context.
2952	*
2953	* @remarks No-long-jump zone!!!
2954	*/
2955	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2956	{
2957	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2958	return VINF_SUCCESS;
2959
2960	#ifdef DEBUG
2961	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2962	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2963	{
2964	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2965	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2966	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2967	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2968	}
2969	#endif
2970
2971	int rc = VERR_INTERNAL_ERROR_5;
2972	bool fInterceptDB = false;
2973	bool fInterceptMovDRx = false;
2974	if (DBGFIsStepping(pVCpu))
2975	{
2976	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2977	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2978	{
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2980	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2981	AssertRCReturn(rc, rc);
2982	Assert(fInterceptDB == false);
2983	}
2984	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2985	{
2986	Assert(fInterceptDB == false);
2987	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2988	* just continuing? */
2989	}
2990	else
2991	fInterceptDB = true;
2992	}
2993	else
2994	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2995
2996	/*
2997	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2998	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2999	* The same for the hypervisor DRx registers, priority is for the guest here.
3000	*/
3001	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3002	&& !CPUMIsGuestDebugStateActive(pVCpu))
3003	{
3004	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3005	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3006	AssertRCReturn(rc, rc);
3007	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3008	Assert(fInterceptMovDRx == false);
3009	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3010	}
3011	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3012	&& !CPUMIsHyperDebugStateActive(pVCpu))
3013	{
3014	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3015	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3016	AssertRCReturn(rc, rc);
3017	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3018	fInterceptMovDRx = true;
3019	}
3020	else
3021	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3022
3023	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3024	if (fInterceptDB)
3025	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3026	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3027	{
3028	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3029	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3030	#endif
3031	}
3032
3033	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3034	if (fInterceptMovDRx)
3035	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3036	else
3037	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3038
3039	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3040	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3041
3042	/* The guest's view of its DR7 is unblemished. */
3043	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3044
3045	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3046	return rc;
3047	}
3048
3049
3050	#ifdef DEBUG
3051	/**
3052	* Debug function to validate segment registers.
3053	*/
3054	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3055	{
3056	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3057	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3058	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3059	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3060	{
3061	/** @todo DPL checks for CS, SS. */
3062	/* Protected mode checks */
3063	/* CS */
3064	Assert(pCtx->cs.Attr.n.u1Present);
3065	Assert(!(pCtx->cs.Attr.u & 0xf00));
3066	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3067	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3068	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3069	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3070	\|\| (pCtx->cs.Attr.n.u1Granularity));
3071	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3072	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3073	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3074	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3075	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3076	else
3077	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3078	/* SS */
3079	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3080	{
3081	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3082	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3083	Assert(pCtx->ss.Attr.n.u1Present);
3084	Assert(!(pCtx->ss.Attr.u & 0xf00));
3085	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3086	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3087	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3088	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3089	\|\| (pCtx->ss.Attr.n.u1Granularity));
3090	}
3091	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3092	/* CR0 might not be up-to-date here always, hence disabled. */
3093	#if 0
3094	if (!pCtx->cr0 & X86_CR0_PE)
3095	Assert(!pCtx->ss.Attr.n.u2Dpl);
3096	#endif
3097	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3098	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3099	{
3100	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3101	Assert(pCtx->ds.Attr.n.u1Present);
3102	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3103	Assert(!(pCtx->ds.Attr.u & 0xf00));
3104	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3105	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3106	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3107	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3108	\|\| (pCtx->ds.Attr.n.u1Granularity));
3109	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3110	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3111	}
3112	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3113	{
3114	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3115	Assert(pCtx->es.Attr.n.u1Present);
3116	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3117	Assert(!(pCtx->es.Attr.u & 0xf00));
3118	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3119	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3120	\|\| !(pCtx->es.Attr.n.u1Granularity));
3121	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3122	\|\| (pCtx->es.Attr.n.u1Granularity));
3123	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3124	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3125	}
3126	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3127	{
3128	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3129	Assert(pCtx->fs.Attr.n.u1Present);
3130	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3131	Assert(!(pCtx->fs.Attr.u & 0xf00));
3132	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3133	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3134	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3135	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3136	\|\| (pCtx->fs.Attr.n.u1Granularity));
3137	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3138	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3139	}
3140	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3141	{
3142	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3143	Assert(pCtx->gs.Attr.n.u1Present);
3144	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3145	Assert(!(pCtx->gs.Attr.u & 0xf00));
3146	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3147	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3148	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3149	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3150	\|\| (pCtx->gs.Attr.n.u1Granularity));
3151	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3152	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3153	}
3154	/* 64-bit capable CPUs. */
3155	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3156	Assert(!(pCtx->cs.u64Base >> 32));
3157	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3158	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3159	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3160	# endif
3161	}
3162	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3163	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3164	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3165	{
3166	/* Real and v86 mode checks. */
3167	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3168	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3169	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3170	{
3171	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3172	}
3173	else
3174	{
3175	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3176	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3177	}
3178
3179	/* CS */
3180	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3181	Assert(pCtx->cs.u32Limit == 0xffff);
3182	Assert(u32CSAttr == 0xf3);
3183	/* SS */
3184	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3185	Assert(pCtx->ss.u32Limit == 0xffff);
3186	Assert(u32SSAttr == 0xf3);
3187	/* DS */
3188	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3189	Assert(pCtx->ds.u32Limit == 0xffff);
3190	Assert(u32DSAttr == 0xf3);
3191	/* ES */
3192	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3193	Assert(pCtx->es.u32Limit == 0xffff);
3194	Assert(u32ESAttr == 0xf3);
3195	/* FS */
3196	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3197	Assert(pCtx->fs.u32Limit == 0xffff);
3198	Assert(u32FSAttr == 0xf3);
3199	/* GS */
3200	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3201	Assert(pCtx->gs.u32Limit == 0xffff);
3202	Assert(u32GSAttr == 0xf3);
3203	/* 64-bit capable CPUs. */
3204	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3205	Assert(!(pCtx->cs.u64Base >> 32));
3206	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3207	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3208	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3209	# endif
3210	}
3211	}
3212	#endif /* DEBUG */
3213
3214
3215	/**
3216	* Writes a guest segment register into the guest-state area in the VMCS.
3217	*
3218	* @returns VBox status code.
3219	* @param pVM Pointer to the VM.
3220	* @param pVCpu Pointer to the VMCPU.
3221	* @param idxSel Index of the selector in the VMCS.
3222	* @param idxLimit Index of the segment limit in the VMCS.
3223	* @param idxBase Index of the segment base in the VMCS.
3224	* @param idxAccess Index of the access rights of the segment in the VMCS.
3225	* @param pSelReg Pointer to the segment selector.
3226	* @param pCtx Pointer to the guest-CPU context.
3227	*
3228	* @remarks No-long-jump zone!!!
3229	*/
3230	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3231	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3232	{
3233	int rc;
3234	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3235	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3236	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3237	AssertRCReturn(rc, rc);
3238
3239	uint32_t u32Access = pSelReg->Attr.u;
3240	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3241	{
3242	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3243	u32Access = 0xf3;
3244	Assert(pVM->hm.s.vmx.pRealModeTSS);
3245	Assert(PDMVmmDevHeapIsEnabled(pVM));
3246	}
3247	else
3248	{
3249	/*
3250	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3251	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3252	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3253	* loaded in protected-mode have their attribute as 0.
3254	*/
3255	if (!u32Access)
3256	u32Access = VMX_SEL_UNUSABLE;
3257	}
3258
3259	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3260	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3261	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3262
3263	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3264	AssertRCReturn(rc, rc);
3265	return rc;
3266	}
3267
3268
3269	/**
3270	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3271	* into the guest-state area in the VMCS.
3272	*
3273	* @returns VBox status code.
3274	* @param pVM Pointer to the VM.
3275	* @param pVCPU Pointer to the VMCPU.
3276	* @param pCtx Pointer to the guest-CPU context.
3277	*
3278	* @remarks No-long-jump zone!!!
3279	* @remarks Requires RFLAGS (for debug assertions).
3280	*/
3281	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3282	{
3283	int rc = VERR_INTERNAL_ERROR_5;
3284
3285	/*
3286	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3287	*/
3288	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3289	{
3290	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3291	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3292	{
3293	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3294	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3295	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3296	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3297	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3298	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3299	}
3300
3301	#ifdef VBOX_WITH_REM
3302	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3303	{
3304	Assert(pVM->hm.s.vmx.pRealModeTSS);
3305	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3306	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3307	{
3308	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3309	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3310	&& enmGuestMode >= PGMMODE_PROTECTED)
3311	{
3312	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3313	in real-mode (e.g. OpenBSD 4.0) */
3314	REMFlushTBs(pVM);
3315	Log2(("Switch to protected mode detected!\n"));
3316	}
3317	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3318	}
3319	}
3320	#endif
3321	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3322	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3323	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3324	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3325	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3326	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3327	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3328	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3329	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3330	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3331	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3332	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3333	AssertRCReturn(rc, rc);
3334
3335	#ifdef DEBUG
3336	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3337	#endif
3338	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3339	}
3340
3341	/*
3342	* Guest TR.
3343	*/
3344	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3345	{
3346	/*
3347	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3348	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3349	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3350	*/
3351	uint16_t u16Sel = 0;
3352	uint32_t u32Limit = 0;
3353	uint64_t u64Base = 0;
3354	uint32_t u32AccessRights = 0;
3355
3356	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3357	{
3358	u16Sel = pCtx->tr.Sel;
3359	u32Limit = pCtx->tr.u32Limit;
3360	u64Base = pCtx->tr.u64Base;
3361	u32AccessRights = pCtx->tr.Attr.u;
3362	}
3363	else
3364	{
3365	Assert(pVM->hm.s.vmx.pRealModeTSS);
3366	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3367
3368	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3369	RTGCPHYS GCPhys;
3370	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3371	AssertRCReturn(rc, rc);
3372
3373	X86DESCATTR DescAttr;
3374	DescAttr.u = 0;
3375	DescAttr.n.u1Present = 1;
3376	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3377
3378	u16Sel = 0;
3379	u32Limit = HM_VTX_TSS_SIZE;
3380	u64Base = GCPhys; /* in real-mode phys = virt. */
3381	u32AccessRights = DescAttr.u;
3382	}
3383
3384	/* Validate. */
3385	Assert(!(u16Sel & RT_BIT(2)));
3386	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3387	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3388	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3389	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3390	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3391	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3392	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3393	Assert( (u32Limit & 0xfff) == 0xfff
3394	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3395	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3396	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3397
3398	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3399	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3400	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3401	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3402	AssertRCReturn(rc, rc);
3403
3404	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3405	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3406	}
3407
3408	/*
3409	* Guest GDTR.
3410	*/
3411	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3412	{
3413	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3414	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3415	AssertRCReturn(rc, rc);
3416
3417	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3418	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3419	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3420	}
3421
3422	/*
3423	* Guest LDTR.
3424	*/
3425	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3426	{
3427	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3428	uint32_t u32Access = 0;
3429	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3430	u32Access = VMX_SEL_UNUSABLE;
3431	else
3432	u32Access = pCtx->ldtr.Attr.u;
3433
3434	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3435	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3436	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3437	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3438	AssertRCReturn(rc, rc);
3439
3440	/* Validate. */
3441	if (!(u32Access & VMX_SEL_UNUSABLE))
3442	{
3443	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3444	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3445	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3446	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3447	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3448	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3449	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3450	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3451	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3452	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3453	}
3454
3455	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3456	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3457	}
3458
3459	/*
3460	* Guest IDTR.
3461	*/
3462	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3463	{
3464	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3465	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3466	AssertRCReturn(rc, rc);
3467
3468	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3469	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3470	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3471	}
3472
3473	/*
3474	* Guest FS & GS base MSRs.
3475	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3476	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3477	* are only available in 64-bit mode.
3478	*/
3479	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3480	* should not be necessary to do it in assembly again. */
3481	if (CPUMIsGuestInLongModeEx(pCtx))
3482	{
3483	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3484	{
3485	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3486	AssertRCReturn(rc, rc);
3487	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3488	}
3489
3490	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3491	{
3492	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3493	AssertRCReturn(rc, rc);
3494	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3495	}
3496	}
3497	else
3498	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3499
3500	return VINF_SUCCESS;
3501	}
3502
3503
3504	/**
3505	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3506	* areas. These MSRs will automatically be loaded to the host CPU on every
3507	* successful VM entry and stored from the host CPU on every successful VM exit.
3508	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3509	*
3510	* @returns VBox status code.
3511	* @param pVM Pointer to the VM.
3512	* @param pVCpu Pointer to the VMCPU.
3513	* @param pCtx Pointer to the guest-CPU context.
3514	*
3515	* @remarks No-long-jump zone!!!
3516	*/
3517	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3518	{
3519	AssertPtr(pVCpu);
3520	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3521
3522	/*
3523	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3524	*/
3525	int rc = VINF_SUCCESS;
3526	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3527	{
3528	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3529	unsigned cGuestMsrs = 0;
3530
3531	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3532	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3533	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3534	if (fSupportsNX \|\| fSupportsLongMode)
3535	{
3536	/** @todo support save IA32_EFER, i.e.
3537	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3538	* guest EFER need not be part of the VM-entry MSR-load area. */
3539	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3540	pGuestMsr->u32Reserved = 0;
3541	pGuestMsr->u64Value = pCtx->msrEFER;
3542	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3543	if (!CPUMIsGuestInLongModeEx(pCtx))
3544	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3545	pGuestMsr++; cGuestMsrs++;
3546	if (fSupportsLongMode)
3547	{
3548	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3549	pGuestMsr->u32Reserved = 0;
3550	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3551	pGuestMsr++; cGuestMsrs++;
3552	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3553	pGuestMsr->u32Reserved = 0;
3554	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3555	pGuestMsr++; cGuestMsrs++;
3556	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3557	pGuestMsr->u32Reserved = 0;
3558	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3559	pGuestMsr++; cGuestMsrs++;
3560	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3561	#if 0
3562	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3563	pGuestMsr->u32Reserved = 0;
3564	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3565	pGuestMsr++; cGuestMsrs++;
3566	#endif
3567	}
3568	}
3569
3570	/*
3571	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3572	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3573	*/
3574	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3575	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3576	{
3577	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3578	pGuestMsr->u32Reserved = 0;
3579	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3580	AssertRCReturn(rc, rc);
3581	pGuestMsr++; cGuestMsrs++;
3582	}
3583
3584	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3585	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3586	{
3587	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3588	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3589	}
3590
3591	/* Update the VCPU's copy of the guest MSR count. */
3592	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3593	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3594	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3595	AssertRCReturn(rc, rc);
3596
3597	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3598	}
3599
3600	/*
3601	* Guest Sysenter MSRs.
3602	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3603	* VM exits on WRMSRs for these MSRs.
3604	*/
3605	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3606	{
3607	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3608	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3609	}
3610	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3611	{
3612	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3613	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3614	}
3615	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3616	{
3617	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3618	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3619	}
3620	AssertRCReturn(rc, rc);
3621
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest activity state into the guest-state area in the VMCS.
3628	*
3629	* @returns VBox status code.
3630	* @param pVM Pointer to the VM.
3631	* @param pVCpu Pointer to the VMCPU.
3632	* @param pCtx Pointer to the guest-CPU context.
3633	*
3634	* @remarks No-long-jump zone!!!
3635	*/
3636	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3637	{
3638	/** @todo See if we can make use of other states, e.g.
3639	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3640	int rc = VINF_SUCCESS;
3641	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3642	{
3643	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3644	AssertRCReturn(rc, rc);
3645	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3646	}
3647	return rc;
3648	}
3649
3650
3651	/**
3652	* Sets up the appropriate function to run guest code.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCpu Pointer to the VMCPU.
3657	* @param pCtx Pointer to the guest-CPU context.
3658	*
3659	* @remarks No-long-jump zone!!!
3660	* @todo change this to return void.
3661	*/
3662	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3663	{
3664	if (CPUMIsGuestInLongModeEx(pCtx))
3665	{
3666	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3667	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3668	#endif
3669	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3670	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3671	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3672	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3673	#else
3674	/* 64-bit host or hybrid host. */
3675	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3676	#endif
3677	}
3678	else
3679	{
3680	/* Guest is not in long mode, use the 32-bit handler. */
3681	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3682	}
3683	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3684	return VINF_SUCCESS;
3685	}
3686
3687
3688	/**
3689	* Wrapper for running the guest code in VT-x.
3690	*
3691	* @returns VBox strict status code.
3692	* @param pVM Pointer to the VM.
3693	* @param pVCpu Pointer to the VMCPU.
3694	* @param pCtx Pointer to the guest-CPU context.
3695	*
3696	* @remarks No-long-jump zone!!!
3697	*/
3698	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3699	{
3700	/*
3701	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3702	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3703	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3704	*/
3705	#ifdef VBOX_WITH_KERNEL_USING_XMM
3706	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3707	#else
3708	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3709	#endif
3710	}
3711
3712
3713	/**
3714	* Report world-switch error and dump some useful debug info.
3715	*
3716	* @param pVM Pointer to the VM.
3717	* @param pVCpu Pointer to the VMCPU.
3718	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3719	* @param pCtx Pointer to the guest-CPU context.
3720	* @param pVmxTransient Pointer to the VMX transient structure (only
3721	* exitReason updated).
3722	*/
3723	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3724	{
3725	Assert(pVM);
3726	Assert(pVCpu);
3727	Assert(pCtx);
3728	Assert(pVmxTransient);
3729	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3730
3731	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3732	switch (rcVMRun)
3733	{
3734	case VERR_VMX_INVALID_VMXON_PTR:
3735	AssertFailed();
3736	break;
3737	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3738	case VERR_VMX_UNABLE_TO_START_VM:
3739	case VERR_VMX_UNABLE_TO_RESUME_VM:
3740	{
3741	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3742	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3743	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3744	AssertRC(rc);
3745
3746	#ifdef VBOX_STRICT
3747	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3748	pVmxTransient->uExitReason));
3749	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3750	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3751	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3752	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3753	else
3754	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3755
3756	/* VMX control bits. */
3757	uint32_t u32Val;
3758	uint64_t u64Val;
3759	RTHCUINTREG uHCReg;
3760	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3761	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3762	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3763	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3764	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3765	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3766	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3767	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3768	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3769	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3770	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3771	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3772	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3773	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3774	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3775	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3777	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3778	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3779	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3780	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3781	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3783	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3785	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3786	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3787	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3789	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3790	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3791	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3792	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3793	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3794	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3795	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3796	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3797	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3798	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3799	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3800	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3801	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3802
3803	/* Guest bits. */
3804	RTGCUINTREG uGCReg;
3805	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3806	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3807	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3808	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3809	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3810	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3811	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3813
3814	/* Host bits. */
3815	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3816	Log(("Host CR0 %#RHr\n", uHCReg));
3817	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3818	Log(("Host CR3 %#RHr\n", uHCReg));
3819	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3820	Log(("Host CR4 %#RHr\n", uHCReg));
3821
3822	RTGDTR HostGdtr;
3823	PCX86DESCHC pDesc;
3824	ASMGetGDTR(&HostGdtr);
3825	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3826	Log(("Host CS %#08x\n", u32Val));
3827	if (u32Val < HostGdtr.cbGdt)
3828	{
3829	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3830	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3831	}
3832
3833	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3834	Log(("Host DS %#08x\n", u32Val));
3835	if (u32Val < HostGdtr.cbGdt)
3836	{
3837	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3838	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3839	}
3840
3841	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3842	Log(("Host ES %#08x\n", u32Val));
3843	if (u32Val < HostGdtr.cbGdt)
3844	{
3845	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3846	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3847	}
3848
3849	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3850	Log(("Host FS %#08x\n", u32Val));
3851	if (u32Val < HostGdtr.cbGdt)
3852	{
3853	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3854	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3855	}
3856
3857	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3858	Log(("Host GS %#08x\n", u32Val));
3859	if (u32Val < HostGdtr.cbGdt)
3860	{
3861	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3862	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3863	}
3864
3865	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3866	Log(("Host SS %#08x\n", u32Val));
3867	if (u32Val < HostGdtr.cbGdt)
3868	{
3869	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3870	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3871	}
3872
3873	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3874	Log(("Host TR %#08x\n", u32Val));
3875	if (u32Val < HostGdtr.cbGdt)
3876	{
3877	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3878	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3879	}
3880
3881	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3882	Log(("Host TR Base %#RHv\n", uHCReg));
3883	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3884	Log(("Host GDTR Base %#RHv\n", uHCReg));
3885	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3886	Log(("Host IDTR Base %#RHv\n", uHCReg));
3887	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3888	Log(("Host SYSENTER CS %#08x\n", u32Val));
3889	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3890	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3891	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3892	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3893	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3894	Log(("Host RSP %#RHv\n", uHCReg));
3895	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3896	Log(("Host RIP %#RHv\n", uHCReg));
3897	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3898	if (VMX_IS_64BIT_HOST_MODE())
3899	{
3900	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3901	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3902	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3903	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3904	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3905	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3906	}
3907	# endif
3908	#endif /* VBOX_STRICT */
3909	break;
3910	}
3911
3912	default:
3913	/* Impossible */
3914	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3915	break;
3916	}
3917	NOREF(pVM);
3918	}
3919
3920
3921	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3922	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3923	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3924	#endif
3925
3926	/**
3927	* Executes the specified handler in 64-bit mode.
3928	*
3929	* @returns VBox status code.
3930	* @param pVM Pointer to the VM.
3931	* @param pVCpu Pointer to the VMCPU.
3932	* @param pCtx Pointer to the guest CPU context.
3933	* @param pfnHandler Pointer to the RC handler function.
3934	* @param cbParam Number of parameters.
3935	* @param paParam Array of 32-bit parameters.
3936	*/
3937	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3938	uint32_t *paParam)
3939	{
3940	int rc, rc2;
3941	PHMGLOBLCPUINFO pCpu;
3942	RTHCPHYS HCPhysCpuPage;
3943	RTHCUINTREG uOldEFlags;
3944
3945	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3946	Assert(pfnHandler);
3947	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3948	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3949
3950	#ifdef VBOX_STRICT
3951	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3952	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3953
3954	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3955	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3956	#endif
3957
3958	/* Disable interrupts. */
3959	uOldEFlags = ASMIntDisableFlags();
3960
3961	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3962	RTCPUID idHostCpu = RTMpCpuId();
3963	CPUMR0SetLApic(pVM, idHostCpu);
3964	#endif
3965
3966	pCpu = HMR0GetCurrentCpu();
3967	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
3968
3969	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3970	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
3971
3972	/* Leave VMX Root Mode. */
3973	VMXDisable();
3974
3975	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3976
3977	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3978	CPUMSetHyperEIP(pVCpu, pfnHandler);
3979	for (int i = (int)cbParam - 1; i >= 0; i--)
3980	CPUMPushHyper(pVCpu, paParam[i]);
3981
3982	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3983
3984	/* Call the switcher. */
3985	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
3986	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3987
3988	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
3989	/* Make sure the VMX instructions don't cause #UD faults. */
3990	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3991
3992	/* Re-enter VMX Root Mode */
3993	rc2 = VMXEnable(HCPhysCpuPage);
3994	if (RT_FAILURE(rc2))
3995	{
3996	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3997	ASMSetFlags(uOldEFlags);
3998	return VERR_VMX_VMXON_FAILED;
3999	}
4000
4001	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4002	AssertRC(rc2);
4003	Assert(!(ASMGetFlags() & X86_EFL_IF));
4004	ASMSetFlags(uOldEFlags);
4005	return rc;
4006	}
4007
4008
4009	/**
4010	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4011	* supporting 64-bit guests.
4012	*
4013	* @returns VBox status code.
4014	* @param fResume Whether to VMLAUNCH or VMRESUME.
4015	* @param pCtx Pointer to the guest-CPU context.
4016	* @param pCache Pointer to the VMCS cache.
4017	* @param pVM Pointer to the VM.
4018	* @param pVCpu Pointer to the VMCPU.
4019	*/
4020	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4021	{
4022	uint32_t aParam[6];
4023	PHMGLOBLCPUINFO pCpu = NULL;
4024	RTHCPHYS HCPhysCpuPage = 0;
4025	int rc = VERR_INTERNAL_ERROR_5;
4026
4027	pCpu = HMR0GetCurrentCpu();
4028	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4029
4030	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4031	pCache->uPos = 1;
4032	pCache->interPD = PGMGetInterPaeCR3(pVM);
4033	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4034	#endif
4035
4036	#ifdef DEBUG
4037	pCache->TestIn.HCPhysCpuPage= 0;
4038	pCache->TestIn.HCPhysVmcs = 0;
4039	pCache->TestIn.pCache = 0;
4040	pCache->TestOut.HCPhysVmcs = 0;
4041	pCache->TestOut.pCache = 0;
4042	pCache->TestOut.pCtx = 0;
4043	pCache->TestOut.eflags = 0;
4044	#endif
4045
4046	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4047	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4048	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4049	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4050	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4051	aParam[5] = 0;
4052
4053	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4054	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4055	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4056	#endif
4057	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4058
4059	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4060	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4061	Assert(pCtx->dr[4] == 10);
4062	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4063	#endif
4064
4065	#ifdef DEBUG
4066	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4067	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4068	pVCpu->hm.s.vmx.HCPhysVmcs));
4069	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4070	pCache->TestOut.HCPhysVmcs));
4071	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4072	pCache->TestOut.pCache));
4073	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4074	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4075	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4076	pCache->TestOut.pCtx));
4077	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4078	#endif
4079	return rc;
4080	}
4081
4082
4083	/**
4084	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4085	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4086	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4087	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4088	*
4089	* @returns VBox status code.
4090	* @param pVM Pointer to the VM.
4091	* @param pVCpu Pointer to the VMCPU.
4092	*/
4093	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4094	{
4095	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4096	{ \
4097	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4098	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4099	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4100	}
4101
4102	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4103	{ \
4104	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4105	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4106	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4107	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4108	}
4109
4110	AssertPtr(pVM);
4111	AssertPtr(pVCpu);
4112	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4113
4114	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4115	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4116	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4117	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4118	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4119	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4120	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4121	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4122	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4123
4124	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4125	#if 0
4126	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4127	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4128	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4129	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4130	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4131	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4132	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4133	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4134	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4135	#endif
4136
4137	/* 32-bit guest-state fields. */
4138	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4142	/* Unused 32-bit guest-state fields. */
4143	#if 0
4144	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4145	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4146	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4147	#endif
4148
4149	/* Natural width guest-state fields. */
4150	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4151	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4152	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4153	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4154	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4155	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4156	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4157	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4158	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4159	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4160	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4161	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4162	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4166	/* Unused natural width guest-state fields. */
4167	#if 0
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4169	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4170	#endif
4171	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4172	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4173
4174	if (pVM->hm.s.fNestedPaging)
4175	{
4176	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4177	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4178	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4179	}
4180	else
4181	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4182
4183	#undef VMXLOCAL_INIT_VMCS_SELREG
4184	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4185	return VINF_SUCCESS;
4186	}
4187
4188
4189	/**
4190	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4191	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4192	* darwin, running 64-bit guests).
4193	*
4194	* @returns VBox status code.
4195	* @param pVCpu Pointer to the VMCPU.
4196	* @param idxField The VMCS field encoding.
4197	* @param u64Val 16, 32 or 64 bits value.
4198	*/
4199	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4200	{
4201	int rc;
4202	switch (idxField)
4203	{
4204	/*
4205	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4206	*/
4207	/* 64-bit Control fields. */
4208	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4209	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4210	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4211	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4212	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4213	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4214	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4215	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4216	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4217	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4218	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4219	case VMX_VMCS64_CTRL_EPTP_FULL:
4220	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4221	/* 64-bit Read-only data fields. */
4222	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4223	/* 64-bit Guest-state fields. */
4224	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4225	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4226	case VMX_VMCS64_GUEST_PAT_FULL:
4227	case VMX_VMCS64_GUEST_EFER_FULL:
4228	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4229	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4230	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4231	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4232	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4233	/* 64-bit Host-state fields. */
4234	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4235	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4236	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4237	{
4238	rc = VMXWriteVmcs32(idxField, u64Val);
4239	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4240	break;
4241	}
4242
4243	/*
4244	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4245	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4246	*/
4247	/* Natural-width Guest-state fields. */
4248	case VMX_VMCS_GUEST_CR0:
4249	case VMX_VMCS_GUEST_CR3:
4250	case VMX_VMCS_GUEST_CR4:
4251	case VMX_VMCS_GUEST_ES_BASE:
4252	case VMX_VMCS_GUEST_CS_BASE:
4253	case VMX_VMCS_GUEST_SS_BASE:
4254	case VMX_VMCS_GUEST_DS_BASE:
4255	case VMX_VMCS_GUEST_FS_BASE:
4256	case VMX_VMCS_GUEST_GS_BASE:
4257	case VMX_VMCS_GUEST_LDTR_BASE:
4258	case VMX_VMCS_GUEST_TR_BASE:
4259	case VMX_VMCS_GUEST_GDTR_BASE:
4260	case VMX_VMCS_GUEST_IDTR_BASE:
4261	case VMX_VMCS_GUEST_DR7:
4262	case VMX_VMCS_GUEST_RSP:
4263	case VMX_VMCS_GUEST_RIP:
4264	case VMX_VMCS_GUEST_RFLAGS:
4265	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4266	case VMX_VMCS_GUEST_SYSENTER_ESP:
4267	case VMX_VMCS_GUEST_SYSENTER_EIP:
4268	{
4269	if ((u64Val >> 32ULL) == 0)
4270	{
4271	/* If this field is 64-bit, VT-x will zero out the top bits. */
4272	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4273	}
4274	else
4275	{
4276	/* Assert that only the 32->64 switcher case should ever come here. */
4277	Assert(pVM->hm.s.fAllow64BitGuests);
4278	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4279	}
4280	break;
4281	}
4282
4283	default:
4284	{
4285	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4286	rc = VERR_INVALID_PARAMETER;
4287	break;
4288	}
4289	}
4290	AssertRCReturn(rc, rc);
4291	return rc;
4292	}
4293
4294
4295	/**
4296	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4297	* hosts (except darwin) for 64-bit guests.
4298	*
4299	* @param pVCpu Pointer to the VMCPU.
4300	* @param idxField The VMCS field encoding.
4301	* @param u64Val 16, 32 or 64 bits value.
4302	*/
4303	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4304	{
4305	AssertPtr(pVCpu);
4306	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4307
4308	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4309	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4310
4311	/* Make sure there are no duplicates. */
4312	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4313	{
4314	if (pCache->Write.aField[i] == idxField)
4315	{
4316	pCache->Write.aFieldVal[i] = u64Val;
4317	return VINF_SUCCESS;
4318	}
4319	}
4320
4321	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4322	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4323	pCache->Write.cValidEntries++;
4324	return VINF_SUCCESS;
4325	}
4326
4327
4328	/**
4329	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4330	*
4331	* @param pCache Pointer to the VMCS cache.
4332	*/
4333	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4334	{
4335	AssertPtr(pCache);
4336	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4337	{
4338	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4339	AssertRC(rc, rc);
4340	}
4341	pCache->Write.cValidEntries = 0;
4342	}
4343
4344
4345	/**
4346	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4347	*
4348	* @param pCache Pointer to the VMCS cache.
4349	* @remarks No-long-jump zone!!!
4350	*/
4351	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4352	{
4353	AssertPtr(pCache);
4354	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4355	{
4356	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4357	AssertRC(rc);
4358	}
4359	}
4360	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4361
4362
4363	/**
4364	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4365	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4366	* timer.
4367	*
4368	* @returns VBox status code.
4369	* @param pVM Pointer to the VM.
4370	* @param pVCpu Pointer to the VMCPU.
4371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4372	* out-of-sync. Make sure to update the required fields
4373	* before using them.
4374	* @remarks No-long-jump zone!!!
4375	*/
4376	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4377	{
4378	int rc = VERR_INTERNAL_ERROR_5;
4379	bool fOffsettedTsc = false;
4380	if (pVM->hm.s.vmx.fUsePreemptTimer)
4381	{
4382	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4383
4384	/* Make sure the returned values have sane upper and lower boundaries. */
4385	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4386	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4387	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4388	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4389
4390	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4391	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4392	}
4393	else
4394	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4395
4396	if (fOffsettedTsc)
4397	{
4398	uint64_t u64CurTSC = ASMReadTSC();
4399	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4400	{
4401	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4402	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4403
4404	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4405	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4406	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4407	}
4408	else
4409	{
4410	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4411	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4412	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4413	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4414	}
4415	}
4416	else
4417	{
4418	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4419	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4420	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4421	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4422	}
4423	}
4424
4425
4426	/**
4427	* Determines if an exception is a benign exception. Benign exceptions
4428	* are ones which cannot cause double-faults.
4429	*
4430	* @returns true if the exception is benign, false otherwise.
4431	* @param uVector The exception vector.
4432	*/
4433	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4434	{
4435	switch (uVector)
4436	{
4437	case X86_XCPT_DB:
4438	case X86_XCPT_NMI:
4439	case X86_XCPT_BP:
4440	case X86_XCPT_OF:
4441	case X86_XCPT_BR:
4442	case X86_XCPT_UD:
4443	case X86_XCPT_NM:
4444	case X86_XCPT_CO_SEG_OVERRUN:
4445	case X86_XCPT_MF:
4446	case X86_XCPT_AC:
4447	case X86_XCPT_MC:
4448	case X86_XCPT_XF:
4449	return true;
4450	default:
4451	return false;
4452	}
4453	}
4454
4455
4456	/**
4457	* Determines if an exception is a contributory exception. Contributory
4458	* exceptions are ones which can cause double-faults.
4459	*
4460	* @returns true if the exception is contributory, false otherwise.
4461	* @param uVector The exception vector.
4462	*/
4463	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4464	{
4465	switch (uVector)
4466	{
4467	case X86_XCPT_GP:
4468	case X86_XCPT_SS:
4469	case X86_XCPT_NP:
4470	case X86_XCPT_TS:
4471	case X86_XCPT_DE:
4472	return true;
4473	default:
4474	return false;
4475	}
4476	return false;
4477	}
4478
4479
4480	/**
4481	* Determines if we are intercepting any contributory exceptions.
4482	*
4483	* @returns true if we are intercepting any contributory exception, false
4484	* otherwise.
4485	* @param pVCpu Pointer to the VMCPU.
4486	*/
4487	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4488	{
4489	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4490	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4491	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4492	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4493	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4494	{
4495	return true;
4496	}
4497	return false;
4498	}
4499
4500
4501	/**
4502	* Handle a condition that occurred while delivering an event through the guest
4503	* IDT.
4504	*
4505	* @returns VBox status code (informational error codes included).
4506	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4507	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4508	* continue execution of the guest which will delivery the #DF.
4509	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4510	*
4511	* @param pVM Pointer to the VM.
4512	* @param pVCpu Pointer to the VMCPU.
4513	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4514	* out-of-sync. Make sure to update the required fields
4515	* before using them.
4516	* @param pVmxTransient Pointer to the VMX transient structure.
4517	*
4518	* @remarks No-long-jump zone!!!
4519	*/
4520	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4521	{
4522	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4523	AssertRCReturn(rc, rc);
4524	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4525	{
4526	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4527	AssertRCReturn(rc, rc);
4528
4529	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4530	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4531	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4532
4533	typedef enum
4534	{
4535	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4536	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4537	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4538	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4539	} VMXREFLECTXCPT;
4540
4541	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4542	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4543	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4544	{
4545	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4546	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4547	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4548	&& uExitVector == X86_XCPT_PF))
4549	{
4550	enmReflect = VMXREFLECTXCPT_XCPT;
4551	}
4552	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4553	&& hmR0VmxIsContributoryXcpt(uExitVector))
4554	{
4555	enmReflect = VMXREFLECTXCPT_DF;
4556	}
4557	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4558	&& uIdtVector == X86_XCPT_PF
4559	&& uExitVector == X86_XCPT_PF)
4560	{
4561	pVmxTransient->fVectoringPF = true;
4562	}
4563	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4564	&& uIdtVector == X86_XCPT_PF
4565	&& hmR0VmxIsContributoryXcpt(uExitVector))
4566	{
4567	enmReflect = VMXREFLECTXCPT_DF;
4568	}
4569	else if (uIdtVector == X86_XCPT_DF)
4570	enmReflect = VMXREFLECTXCPT_TF;
4571	}
4572	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4573	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4574	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4575	{
4576	/*
4577	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4578	* (whatever they are) as they reoccur when restarting the instruction.
4579	*/
4580	enmReflect = VMXREFLECTXCPT_XCPT;
4581	}
4582
4583	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4584	switch (enmReflect)
4585	{
4586	case VMXREFLECTXCPT_XCPT:
4587	{
4588	Assert(!pVCpu->hm.s.Event.fPending);
4589	pVCpu->hm.s.Event.fPending = true;
4590	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4591	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4592	{
4593	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4594	AssertRCReturn(rc, rc);
4595	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4596	}
4597	else
4598	pVCpu->hm.s.Event.u32ErrCode = 0;
4599	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4600	break;
4601	}
4602
4603	case VMXREFLECTXCPT_DF:
4604	{
4605	uint32_t u32IntrInfo;
4606	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4607	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4608	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4609
4610	Assert(!pVCpu->hm.s.Event.fPending);
4611	pVCpu->hm.s.Event.fPending = true;
4612	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4613	pVCpu->hm.s.Event.u32ErrCode = 0;
4614	rc = VINF_VMX_DOUBLE_FAULT;
4615	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4616	break;
4617	}
4618
4619	case VMXREFLECTXCPT_TF:
4620	{
4621	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4622	rc = VINF_EM_RESET;
4623	break;
4624	}
4625
4626	default: /* shut up gcc. */
4627	break;
4628	}
4629	}
4630	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4631	return rc;
4632	}
4633
4634
4635	/**
4636	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4637	*
4638	* @returns VBox status code.
4639	* @param pVM Pointer to the VM.
4640	* @param pVCpu Pointer to the VMCPU.
4641	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4642	* out-of-sync. Make sure to update the required fields
4643	* before using them.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4648	{
4649	int rc = VINF_SUCCESS;
4650	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4651	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4652	{
4653	RTGCUINTREG uVal = 0;
4654	RTCCUINTREG uShadow = 0;
4655	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4656	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4657	AssertRCReturn(rc, rc);
4658	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4659	CPUMSetGuestCR0(pVCpu, uVal);
4660	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4661	}
4662	return rc;
4663	}
4664
4665
4666	/**
4667	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4668	*
4669	* @returns VBox status code.
4670	* @param pVM Pointer to the VM.
4671	* @param pVCpu Pointer to the VMCPU.
4672	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4673	* out-of-sync. Make sure to update the required fields
4674	* before using them.
4675	*
4676	* @remarks No-long-jump zone!!!
4677	*/
4678	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4679	{
4680	int rc = VINF_SUCCESS;
4681	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4682	{
4683	RTGCUINTREG uVal = 0;
4684	RTCCUINTREG uShadow = 0;
4685	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4686	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4687	AssertRCReturn(rc, rc);
4688	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4689	CPUMSetGuestCR4(pVCpu, uVal);
4690	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4691	}
4692	return rc;
4693	}
4694
4695
4696	/**
4697	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4698	*
4699	* @returns VBox status code.
4700	* @param pVM Pointer to the VM.
4701	* @param pVCpu Pointer to the VMCPU.
4702	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4703	* out-of-sync. Make sure to update the required fields
4704	* before using them.
4705	*
4706	* @remarks No-long-jump zone!!!
4707	*/
4708	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4709	{
4710	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4711	return VINF_SUCCESS;
4712
4713	RTGCUINTREG uVal = 0;
4714	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4715	AssertRCReturn(rc, rc);
4716	pMixedCtx->rip = uVal;
4717	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4718	return rc;
4719	}
4720
4721
4722	/**
4723	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4724	*
4725	* @returns VBox status code.
4726	* @param pVM Pointer to the VM.
4727	* @param pVCpu Pointer to the VMCPU.
4728	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4729	* out-of-sync. Make sure to update the required fields
4730	* before using them.
4731	*
4732	* @remarks No-long-jump zone!!!
4733	*/
4734	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735	{
4736	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4737	return VINF_SUCCESS;
4738
4739	RTGCUINTREG uVal = 0;
4740	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4741	AssertRCReturn(rc, rc);
4742	pMixedCtx->rsp = uVal;
4743	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4744	return rc;
4745	}
4746
4747
4748	/**
4749	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4750	*
4751	* @returns VBox status code.
4752	* @param pVM Pointer to the VM.
4753	* @param pVCpu Pointer to the VMCPU.
4754	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4755	* out-of-sync. Make sure to update the required fields
4756	* before using them.
4757	*
4758	* @remarks No-long-jump zone!!!
4759	*/
4760	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4761	{
4762	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4763	return VINF_SUCCESS;
4764
4765	RTGCUINTREG uVal = 0;
4766	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4767	AssertRCReturn(rc, rc);
4768	pMixedCtx->rflags.u64 = uVal;
4769
4770	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4771	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4772	{
4773	Assert(pVM->hm.s.vmx.pRealModeTSS);
4774	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4775	pMixedCtx->eflags.Bits.u1VM = 0;
4776	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4777	}
4778
4779	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4780	return rc;
4781	}
4782
4783
4784	/**
4785	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4786	* guest-CPU context.
4787	*/
4788	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4789	{
4790	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4791	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4792	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4793	AssertRCReturn(rc, rc);
4794	return rc;
4795	}
4796
4797
4798	/**
4799	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4800	* from the guest-state area in the VMCS.
4801	*
4802	* @param pVM Pointer to the VM.
4803	* @param pVCpu Pointer to the VMCPU.
4804	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4805	* out-of-sync. Make sure to update the required fields
4806	* before using them.
4807	*
4808	* @remarks No-long-jump zone!!!
4809	*/
4810	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811	{
4812	uint32_t uIntrState = 0;
4813	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4814	AssertRC(rc);
4815
4816	if (!uIntrState)
4817	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4818	else
4819	{
4820	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4821	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4822	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4823	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4824	AssertRC(rc);
4825	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4826	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4827	}
4828	}
4829
4830
4831	/**
4832	* Saves the guest's activity state.
4833	*
4834	* @returns VBox status code.
4835	* @param pVM Pointer to the VM.
4836	* @param pVCpu Pointer to the VMCPU.
4837	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4838	* out-of-sync. Make sure to update the required fields
4839	* before using them.
4840	*
4841	* @remarks No-long-jump zone!!!
4842	*/
4843	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4844	{
4845	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4846	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4847	return VINF_SUCCESS;
4848	}
4849
4850
4851	/**
4852	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4853	* the current VMCS into the guest-CPU context.
4854	*
4855	* @returns VBox status code.
4856	* @param pVM Pointer to the VM.
4857	* @param pVCpu Pointer to the VMCPU.
4858	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4859	* out-of-sync. Make sure to update the required fields
4860	* before using them.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4865	{
4866	int rc = VINF_SUCCESS;
4867	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4868	{
4869	uint32_t u32Val = 0;
4870	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4871	pMixedCtx->SysEnter.cs = u32Val;
4872	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4873	}
4874
4875	RTGCUINTREG uGCVal = 0;
4876	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4877	{
4878	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4879	pMixedCtx->SysEnter.eip = uGCVal;
4880	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4881	}
4882	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4883	{
4884	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4885	pMixedCtx->SysEnter.esp = uGCVal;
4886	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4887	}
4888	return rc;
4889	}
4890
4891
4892	/**
4893	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4894	* context.
4895	*
4896	* @returns VBox status code.
4897	* @param pVM Pointer to the VM.
4898	* @param pVCpu Pointer to the VMCPU.
4899	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4900	* out-of-sync. Make sure to update the required fields
4901	* before using them.
4902	*
4903	* @remarks No-long-jump zone!!!
4904	*/
4905	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4906	{
4907	RTGCUINTREG uVal = 0;
4908	int rc = VINF_SUCCESS;
4909	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4910	{
4911	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4912	pMixedCtx->fs.u64Base = uVal;
4913	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4914	}
4915	return rc;
4916	}
4917
4918
4919	/**
4920	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4921	* context.
4922	*
4923	* @returns VBox status code.
4924	* @param pVM Pointer to the VM.
4925	* @param pVCpu Pointer to the VMCPU.
4926	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4927	* out-of-sync. Make sure to update the required fields
4928	* before using them.
4929	*
4930	* @remarks No-long-jump zone!!!
4931	*/
4932	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4933	{
4934	RTGCUINTREG uVal = 0;
4935	int rc = VINF_SUCCESS;
4936	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4937	{
4938	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4939	pMixedCtx->gs.u64Base = uVal;
4940	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4941	}
4942	return rc;
4943	}
4944
4945
4946	/**
4947	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4948	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4949	*
4950	* @returns VBox status code.
4951	* @param pVM Pointer to the VM.
4952	* @param pVCpu Pointer to the VMCPU.
4953	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4954	* out-of-sync. Make sure to update the required fields
4955	* before using them.
4956	*
4957	* @remarks No-long-jump zone!!!
4958	*/
4959	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4960	{
4961	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
4962	return VINF_SUCCESS;
4963
4964	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
4965	{
4966	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4967	pMsr += i;
4968	switch (pMsr->u32IndexMSR)
4969	{
4970	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
4971	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
4972	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
4973	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
4974	#if 0
4975	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
4976	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
4977	#endif
4978	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
4979	default:
4980	{
4981	AssertFailed();
4982	return VERR_HM_UNEXPECTED_LD_ST_MSR;
4983	}
4984	}
4985	}
4986	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
4987	return VINF_SUCCESS;
4988	}
4989
4990
4991	/**
4992	* Saves the guest control registers from the current VMCS into the guest-CPU
4993	* context.
4994	*
4995	* @returns VBox status code.
4996	* @param pVM Pointer to the VM.
4997	* @param pVCpu Pointer to the VMCPU.
4998	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4999	* out-of-sync. Make sure to update the required fields
5000	* before using them.
5001	*
5002	* @remarks No-long-jump zone!!!
5003	*/
5004	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5005	{
5006	RTGCUINTREG uVal = 0;
5007	RTGCUINTREG uShadow = 0;
5008	int rc = VINF_SUCCESS;
5009
5010	/* Guest CR0. Guest FPU. */
5011	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5012	AssertRCReturn(rc, rc);
5013
5014	/* Guest CR4. */
5015	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5016	AssertRCReturn(rc, rc);
5017
5018	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5019	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5020	{
5021	if ( pVM->hm.s.fNestedPaging
5022	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5023	{
5024	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5025	AssertRCReturn(rc, rc);
5026	if (pMixedCtx->cr3 != uVal)
5027	{
5028	CPUMSetGuestCR3(pVCpu, uVal);
5029	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5030	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5031	}
5032
5033	/* We require EFER to check PAE mode. */
5034	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5035	AssertRCReturn(rc, rc);
5036
5037	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5038	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5039	{
5040	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5041	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5042	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5043	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5044	AssertRCReturn(rc, rc);
5045	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5046	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5047	}
5048	}
5049	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5050	}
5051	return rc;
5052	}
5053
5054
5055	/**
5056	* Reads a guest segment register from the current VMCS into the guest-CPU
5057	* context.
5058	*
5059	* @returns VBox status code.
5060	* @param idxSel Index of the selector in the VMCS.
5061	* @param idxLimit Index of the segment limit in the VMCS.
5062	* @param idxBase Index of the segment base in the VMCS.
5063	* @param idxAccess Index of the access rights of the segment in the VMCS.
5064	* @param pSelReg Pointer to the segment selector.
5065	*
5066	* @remarks No-long-jump zone!!!
5067	*/
5068	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5069	PCPUMSELREG pSelReg)
5070	{
5071	uint32_t u32Val = 0;
5072	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5073	pSelReg->Sel = (uint16_t)u32Val;
5074	pSelReg->ValidSel = (uint16_t)u32Val;
5075	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5076
5077	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5078	pSelReg->u32Limit = u32Val;
5079
5080	RTGCUINTREG uGCVal = 0;
5081	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5082	pSelReg->u64Base = uGCVal;
5083
5084	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5085	pSelReg->Attr.u = u32Val;
5086
5087	/*
5088	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5089	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5090	*/
5091	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5092	{
5093	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5094	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5095	}
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Saves the guest segment registers from the current VMCS into the guest-CPU
5102	* context.
5103	*
5104	* @returns VBox status code.
5105	* @param pVM Pointer to the VM.
5106	* @param pVCpu Pointer to the VMCPU.
5107	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5108	* out-of-sync. Make sure to update the required fields
5109	* before using them.
5110	*
5111	* @remarks No-long-jump zone!!!
5112	*/
5113	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114	{
5115	int rc = VINF_SUCCESS;
5116
5117	/* Guest segment registers. */
5118	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5119	{
5120	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5121	AssertRCReturn(rc, rc);
5122
5123	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5124	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5125	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5126	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5127	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5128	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5129	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5130	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5131	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5132	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5133	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5134	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5135	AssertRCReturn(rc, rc);
5136
5137	/* Restore segment attributes for real-on-v86 mode hack. */
5138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5139	{
5140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5146	}
5147
5148	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5149	}
5150
5151	/* Guest LDTR. */
5152	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5153	{
5154	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5155	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5156	AssertRCReturn(rc, rc);
5157	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5158	}
5159
5160	/* Guest GDTR. */
5161	RTGCUINTREG uGCVal = 0;
5162	uint32_t u32Val = 0;
5163	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5164	{
5165	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5166	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5167	pMixedCtx->gdtr.pGdt = uGCVal;
5168	pMixedCtx->gdtr.cbGdt = u32Val;
5169	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5170	}
5171
5172	/* Guest IDTR. */
5173	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5174	{
5175	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5176	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5177	pMixedCtx->idtr.pIdt = uGCVal;
5178	pMixedCtx->idtr.cbIdt = u32Val;
5179	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5180	}
5181
5182	/* Guest TR. */
5183	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5184	{
5185	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5186	AssertRCReturn(rc, rc);
5187
5188	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5189	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5190	{
5191	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5192	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5193	AssertRCReturn(rc, rc);
5194	}
5195	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5196	}
5197	return rc;
5198	}
5199
5200
5201	/**
5202	* Saves the guest debug registers from the current VMCS into the guest-CPU
5203	* context.
5204	*
5205	* @returns VBox status code.
5206	* @param pVM Pointer to the VM.
5207	* @param pVCpu Pointer to the VMCPU.
5208	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5209	* out-of-sync. Make sure to update the required fields
5210	* before using them.
5211	*
5212	* @remarks No-long-jump zone!!!
5213	*/
5214	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5215	{
5216	int rc = VINF_SUCCESS;
5217	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5218	{
5219	RTGCUINTREG uVal;
5220	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5221	pMixedCtx->dr[7] = uVal;
5222
5223	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5224	}
5225	return rc;
5226	}
5227
5228
5229	/**
5230	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5231	*
5232	* @returns VBox status code.
5233	* @param pVM Pointer to the VM.
5234	* @param pVCpu Pointer to the VMCPU.
5235	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5236	* out-of-sync. Make sure to update the required fields
5237	* before using them.
5238	*
5239	* @remarks No-long-jump zone!!!
5240	*/
5241	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5242	{
5243	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5244	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5245	return VINF_SUCCESS;
5246	}
5247
5248
5249	/**
5250	* Saves the entire guest state from the currently active VMCS into the
5251	* guest-CPU context. This essentially VMREADs all guest-data.
5252	*
5253	* @returns VBox status code.
5254	* @param pVM Pointer to the VM.
5255	* @param pVCpu Pointer to the VMCPU.
5256	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5257	* out-of-sync. Make sure to update the required fields
5258	* before using them.
5259	*/
5260	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5261	{
5262	Assert(pVM);
5263	Assert(pVCpu);
5264	Assert(pMixedCtx);
5265
5266	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5267	return VINF_SUCCESS;
5268
5269	VMMRZCallRing3Disable(pVCpu); /* We're not using hmR0VmxCallRing3Disable() as we want to respect nesting here. */
5270
5271	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5272	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5273
5274	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5275	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5276
5277	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5278	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5279
5280	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5281	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5282
5283	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5284	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5285
5286	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5287	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5288
5289	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5290	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5291
5292	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5293	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5294
5295	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5296	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5297
5298	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5299	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5300
5301	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5302	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5303
5304	VMMRZCallRing3Enable(pVCpu);
5305	return rc;
5306	}
5307
5308
5309	/**
5310	* Check per-VM and per-VCPU force flag actions that require us to go back to
5311	* ring-3 for one reason or another.
5312	*
5313	* @returns VBox status code (information status code included).
5314	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5315	* ring-3.
5316	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5317	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5318	* interrupts)
5319	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5320	* all EMTs to be in ring-3.
5321	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5322	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5323	* to the EM loop.
5324	*
5325	* @param pVM Pointer to the VM.
5326	* @param pVCpu Pointer to the VMCPU.
5327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5328	* out-of-sync. Make sure to update the required fields
5329	* before using them.
5330	*/
5331	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5332	{
5333	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5334
5335	int rc = VERR_INTERNAL_ERROR_5;
5336	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5337	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5338	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5339	{
5340	/* We need the control registers now, make sure the guest-CPU context is updated. */
5341	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5342	AssertRCReturn(rc, rc);
5343
5344	/* Pending HM CR3 sync. */
5345	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5346	{
5347	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5348	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5349	}
5350	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5351	{
5352	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5353	AssertRCReturn(rc, rc);
5354	}
5355
5356	/* Pending PGM C3 sync. */
5357	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5358	{
5359	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5360	if (rc != VINF_SUCCESS)
5361	{
5362	AssertRC(rc);
5363	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5364	return rc;
5365	}
5366	}
5367
5368	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5369	/* -XXX- what was that about single stepping? */
5370	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5371	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5372	{
5373	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5374	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5375	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5376	return rc;
5377	}
5378
5379	/* Pending VM request packets, such as hardware interrupts. */
5380	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5381	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5382	{
5383	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5384	return VINF_EM_PENDING_REQUEST;
5385	}
5386
5387	/* Pending PGM pool flushes. */
5388	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5389	{
5390	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5391	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5392	}
5393
5394	/* Pending DMA requests. */
5395	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5396	{
5397	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5398	return VINF_EM_RAW_TO_R3;
5399	}
5400	}
5401
5402	/* Paranoia. */
5403	Assert(rc != VERR_EM_INTERPRETER);
5404	return VINF_SUCCESS;
5405	}
5406
5407
5408	/**
5409	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5410	* VT-x to execute any instruction.
5411	*
5412	* @param pvCpu Pointer to the VMCPU.
5413	*/
5414	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5415	{
5416	if (pVCpu->hm.s.Event.fPending)
5417	{
5418	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5419	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5420	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5421	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5422
5423	/* If a trap was already pending, we did something wrong! */
5424	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5425
5426	/* A page-fault exception during a page-fault would become a double-fault. */
5427	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5428	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5429
5430	TRPMEVENT enmTrapType;
5431	switch (uVectorType)
5432	{
5433	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5434	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5435	enmTrapType = TRPM_HARDWARE_INT;
5436	break;
5437	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5438	enmTrapType = TRPM_SOFTWARE_INT;
5439	break;
5440	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5441	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5442	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5443	enmTrapType = TRPM_TRAP;
5444	break;
5445	default:
5446	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5447	enmTrapType = TRPM_32BIT_HACK;
5448	break;
5449	}
5450	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5451	AssertRC(rc);
5452	if (fErrorCodeValid)
5453	TRPMSetErrorCode(pVCpu, uErrorCode);
5454
5455	/* Clear the VT-x state bits now that TRPM has the information. */
5456	pVCpu->hm.s.Event.fPending = false;
5457	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5458	AssertRC(rc);
5459	}
5460	}
5461
5462
5463	/**
5464	* Does the necessary state syncing before doing a longjmp to ring-3.
5465	*
5466	* @param pVM Pointer to the VM.
5467	* @param pVCpu Pointer to the VMCPU.
5468	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5469	* out-of-sync. Make sure to update the required fields
5470	* before using them.
5471	* @param rcExit The reason for exiting to ring-3. Can be
5472	* VINF_VMM_UNKNOWN_RING3_CALL.
5473	*
5474	* @remarks No-long-jmp zone!!!
5475	*/
5476	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5477	{
5478	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5479
5480	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5481	AssertRC(rc);
5482
5483	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5484	if (CPUMIsGuestFPUStateActive(pVCpu))
5485	{
5486	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5487	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5488	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5489	}
5490
5491	/* Restore debug registers if necessary and resync on next R0 reentry. */
5492	if (CPUMIsGuestDebugStateActive(pVCpu))
5493	{
5494	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5495	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5496	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5497	}
5498	else if (CPUMIsHyperDebugStateActive(pVCpu))
5499	{
5500	CPUMR0LoadHostDebugState(pVM, pVCpu);
5501	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5502	}
5503
5504	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5505	}
5506
5507
5508	/**
5509	* An action requires us to go back to ring-3. This function does the necessary
5510	* steps before we can safely return to ring-3. This is not the same as longjmps
5511	* to ring-3, this is voluntary.
5512	*
5513	* @param pVM Pointer to the VM.
5514	* @param pVCpu Pointer to the VMCPU.
5515	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5516	* out-of-sync. Make sure to update the required fields
5517	* before using them.
5518	* @param rcExit The reason for exiting to ring-3. Can be
5519	* VINF_VMM_UNKNOWN_RING3_CALL.
5520	*/
5521	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5522	{
5523	Assert(pVM);
5524	Assert(pVCpu);
5525	Assert(pMixedCtx);
5526	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5527
5528	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5529	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5530	return;
5531
5532	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5533	VMMRZCallRing3Disable(pVCpu);
5534	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5535
5536	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5537	hmR0VmxUpdateTRPMTrap(pVCpu);
5538
5539	/* Sync. the guest state. */
5540	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5541	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5542
5543	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5544	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5545	\| CPUM_CHANGED_LDTR
5546	\| CPUM_CHANGED_GDTR
5547	\| CPUM_CHANGED_IDTR
5548	\| CPUM_CHANGED_TR
5549	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5550
5551	/* On our way back from ring-3 the following needs to be done. */
5552	/** @todo This can change with preemption hooks. */
5553	if (rcExit == VINF_EM_RAW_INTERRUPT)
5554	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5555	else
5556	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5557
5558	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5559	VMMRZCallRing3Enable(pVCpu);
5560	}
5561
5562
5563	/**
5564	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5565	* longjump to ring-3 and possibly get preempted.
5566	*
5567	* @param pVCpu Pointer to the VMCPU.
5568	* @param enmOperation The operation causing the ring-3 longjump.
5569	* @param pvUser The user argument (pointer to the possibly
5570	* out-of-date guest-CPU context).
5571	*
5572	* @remarks Must never be called with @a enmOperation ==
5573	* VMMCALLRING3_VM_R0_ASSERTION.
5574	*/
5575	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5576	{
5577	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5578	Assert(pVCpu);
5579	Assert(pvUser);
5580	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5582
5583	VMMRZCallRing3Disable(pVCpu);
5584	Log(("hmR0VmxLongJmpToRing3\n"));
5585	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5586	VMMRZCallRing3Enable(pVCpu);
5587	}
5588
5589
5590	/**
5591	* Injects any pending TRPM trap into the VM by updating the VMCS.
5592	*
5593	* @returns VBox status code (informational status code included).
5594	* @param pVM Pointer to the VM.
5595	* @param pVCpu Pointer to the VMCPU.
5596	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5597	* out-of-sync. Make sure to update the required fields
5598	* before using them.
5599	*/
5600	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5601	{
5602	if (!TRPMHasTrap(pVCpu))
5603	return VINF_SUCCESS;
5604
5605	uint8_t u8Vector = 0;
5606	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5607	RTGCUINT uErrCode = 0;
5608
5609	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5610	AssertRCReturn(rc, rc);
5611	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5612
5613	rc = TRPMResetTrap(pVCpu);
5614	AssertRCReturn(rc, rc);
5615
5616	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5617	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5618	if (enmTrpmEvent == TRPM_TRAP)
5619	{
5620	switch (u8Vector)
5621	{
5622	case X86_XCPT_BP:
5623	case X86_XCPT_OF:
5624	{
5625	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5626	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5627	break;
5628	}
5629
5630	case X86_XCPT_DF:
5631	case X86_XCPT_TS:
5632	case X86_XCPT_NP:
5633	case X86_XCPT_SS:
5634	case X86_XCPT_GP:
5635	case X86_XCPT_PF:
5636	case X86_XCPT_AC:
5637	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5638	them which VT-x/VMM pushes to the guest stack. */
5639	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5640	/* no break! */
5641	default:
5642	{
5643	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5644	break;
5645	}
5646	}
5647	}
5648	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5649	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5650	else
5651	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5652
5653	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5654	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5655	}
5656
5657
5658	/**
5659	* Checks if there are any pending guest interrupts to be delivered and injects
5660	* them into the VM by updating the VMCS.
5661	*
5662	* @returns VBox status code (informational status codes included).
5663	* @param pVM Pointer to the VM.
5664	* @param pVCpu Pointer to the VMCPU.
5665	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5666	* out-of-sync. Make sure to update the required fields
5667	* before using them.
5668	*
5669	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5670	*/
5671	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5672	{
5673	/* First inject any pending HM interrupts. */
5674	if (pVCpu->hm.s.Event.fPending)
5675	{
5676	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5677	pVCpu->hm.s.Event.u32ErrCode);
5678	AssertRCReturn(rc, rc);
5679	pVCpu->hm.s.Event.fPending = false;
5680	return rc;
5681	}
5682
5683	/** @todo SMI. SMIs take priority over NMIs. */
5684
5685	/* NMI. NMIs take priority over regular interrupts . */
5686	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5687	{
5688	/* Construct an NMI interrupt and inject it into the VMCS. */
5689	RTGCUINTPTR uIntrInfo;
5690	uIntrInfo = X86_XCPT_NMI;
5691	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5692	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5693	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5694	AssertRCReturn(rc, rc);
5695	return rc;
5696	}
5697
5698	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5699	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5700	AssertRCReturn(rc, rc);
5701
5702	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5703	if (!TRPMHasTrap(pVCpu))
5704	{
5705	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5706	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5707	{
5708	/*
5709	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5710	* the PDM interrupt into a TRPM event and inject it.
5711	*/
5712	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5713	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5714	{
5715	uint8_t u8Interrupt = 0;
5716	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5717	if (RT_SUCCESS(rc))
5718	{
5719	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5720	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5721	AssertRCReturn(rc, rc);
5722	}
5723	else
5724	{
5725	/** @todo Does this actually happen? If not turn it into an assertion. */
5726	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5727	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5728	}
5729	}
5730	else if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5731	{
5732	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5733	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5735	AssertRCReturn(rc, rc);
5736	}
5737	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5738	}
5739	}
5740
5741	/* If interrupts can be delivered, inject it into the VM. */
5742	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5743	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5744	&& TRPMHasTrap(pVCpu))
5745	{
5746	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5747	AssertRCReturn(rc, rc);
5748	}
5749	return rc;
5750	}
5751
5752	/**
5753	* Injects an invalid-opcode (#UD) exception into the VM.
5754	*
5755	* @returns VBox status code (informational status code included).
5756	* @param pVM Pointer to the VM.
5757	* @param pVCpu Pointer to the VMCPU.
5758	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5759	* out-of-sync. Make sure to update the required fields
5760	* before using them.
5761	*/
5762	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5763	{
5764	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5765	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5766	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5767	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5768	}
5769
5770
5771	/**
5772	* Injects a double-fault (#DF) exception into the VM.
5773	*
5774	* @returns VBox status code (informational status code included).
5775	* @param pVM Pointer to the VM.
5776	* @param pVCpu Pointer to the VMCPU.
5777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778	* out-of-sync. Make sure to update the required fields
5779	* before using them.
5780	*/
5781	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5782	{
5783	/* Inject the double-fault. */
5784	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5785	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5786	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5787	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5788	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5789	}
5790
5791
5792	/**
5793	* Injects a debug (#DB) exception into the VM.
5794	*
5795	* @returns VBox status code (informational status code included).
5796	* @param pVM Pointer to the VM.
5797	* @param pVCpu Pointer to the VMCPU.
5798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5799	* out-of-sync. Make sure to update the required fields
5800	* before using them.
5801	*/
5802	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5803	{
5804	/* Inject the debug-exception. */
5805	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5806	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5807	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5808	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5809	}
5810
5811
5812	/**
5813	* Injects a overflow (#OF) exception into the VM.
5814	*
5815	* @returns VBox status code (informational status code included).
5816	* @param pVM Pointer to the VM.
5817	* @param pVCpu Pointer to the VMCPU.
5818	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5819	* out-of-sync. Make sure to update the required fields
5820	* before using them.
5821	* @param cbInstr The value of RIP that is to be pushed on the guest
5822	* stack.
5823	*/
5824	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5825	{
5826	/* Inject the overflow exception. */
5827	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5828	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5829	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5830	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5831	}
5832
5833
5834	/**
5835	* Injects a general-protection (#GP) fault into the VM.
5836	*
5837	* @returns VBox status code (informational status code included).
5838	* @param pVM Pointer to the VM.
5839	* @param pVCpu Pointer to the VMCPU.
5840	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5841	* out-of-sync. Make sure to update the required fields
5842	* before using them.
5843	* @param u32ErrorCode The error code associated with the #GP.
5844	*/
5845	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5846	{
5847	/* Inject the general-protection fault. */
5848	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5849	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5850	if (fErrorCodeValid)
5851	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5852	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5853	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5854	}
5855
5856
5857	/**
5858	* Injects a software interrupt (INTn) into the VM.
5859	*
5860	* @returns VBox status code (informational status code included).
5861	* @param pVM Pointer to the VM.
5862	* @param pVCpu Pointer to the VMCPU.
5863	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5864	* out-of-sync. Make sure to update the required fields
5865	* before using them.
5866	* @param uVector The software interrupt vector number.
5867	* @param cbInstr The value of RIP that is to be pushed on the guest
5868	* stack.
5869	*/
5870	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5871	{
5872	/* Inject the INTn. */
5873	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5874	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5875	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5876	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5877	}
5878
5879
5880	/**
5881	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5882	* stack.
5883	*
5884	* @returns VBox status code (information status code included).
5885	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5886	* @param pVM Pointer to the VM.
5887	* @param pMixedCtx Pointer to the guest-CPU context.
5888	* @param uValue The value to push to the guest stack.
5889	*/
5890	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5891	{
5892	/*
5893	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5894	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5895	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5896	*/
5897	if (pMixedCtx->sp == 1)
5898	return VINF_EM_RESET;
5899	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5900	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5901	AssertRCReturn(rc, rc);
5902	return rc;
5903	}
5904
5905
5906	/**
5907	* Injects an event into the guest upon VM-entry by updating the relevant fields
5908	* in the VM-entry area in the VMCS.
5909	*
5910	* @returns VBox status code (informational error codes included).
5911	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5912	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5913	*
5914	* @param pVM Pointer to the VM.
5915	* @param pVCpu Pointer to the VMCPU.
5916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5917	* out-of-sync. Make sure to update the required fields
5918	* before using them.
5919	* @param u64IntrInfo The VM-entry interruption-information field.
5920	* @param cbInstr The VM-entry instruction length in bytes (for software
5921	* interrupts, exceptions and privileged software
5922	* exceptions).
5923	* @param u32ErrCode The VM-entry exception error code.
5924	*
5925	* @remarks No-long-jump zone!!!
5926	*/
5927	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5928	uint32_t u32ErrCode)
5929	{
5930	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5931	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5932	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5933
5934	/* We require CR0 to check if the guest is in real-mode. */
5935	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5936	AssertRCReturn(rc, rc);
5937
5938	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5939	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5940
5941	/*
5942	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5943	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5944	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5945	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5946	*/
5947	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5948	{
5949	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5950	{
5951	Assert(PDMVmmDevHeapIsEnabled(pVM));
5952	Assert(pVM->hm.s.vmx.pRealModeTSS);
5953
5954	/* Save the required guest state bits from the VMCS. */
5955	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5956	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5957	AssertRCReturn(rc, rc);
5958
5959	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
5960	const size_t cbIdtEntry = 4;
5961	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
5962	{
5963	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
5964	if (uVector == X86_XCPT_DF)
5965	return VINF_EM_RESET;
5966	else if (uVector == X86_XCPT_GP)
5967	{
5968	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
5969	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
5970	}
5971
5972	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
5973	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
5974	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
5975	}
5976
5977	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
5978	uint16_t uGuestIp = pMixedCtx->ip;
5979	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
5980	{
5981	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
5982	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
5983	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
5984	}
5985	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
5986	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
5987
5988	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
5989	uint16_t offIdtEntry = 0;
5990	RTSEL selIdtEntry = 0;
5991	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
5992	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
5993	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
5994	AssertRCReturn(rc, rc);
5995
5996	/* Construct the stack frame for the interrupt/exception handler. */
5997	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
5998	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
5999	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6000	AssertRCReturn(rc, rc);
6001
6002	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6003	if (rc == VINF_SUCCESS)
6004	{
6005	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6006	pMixedCtx->rip = offIdtEntry;
6007	pMixedCtx->cs.Sel = selIdtEntry;
6008	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6009
6010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6011	\| HM_CHANGED_GUEST_RIP
6012	\| HM_CHANGED_GUEST_RFLAGS
6013	\| HM_CHANGED_GUEST_RSP;
6014	AssertRCReturn(rc, rc);
6015	}
6016	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6017	return rc;
6018	}
6019	else
6020	{
6021	/*
6022	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6023	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6024	*/
6025	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6026	}
6027	}
6028
6029	/* Add the valid bit, maybe the caller was lazy. */
6030	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6031
6032	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6033	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6034	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6035
6036	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6037	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6038	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6039	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6040	AssertRCReturn(rc, rc);
6041
6042	return rc;
6043	}
6044
6045
6046	/**
6047	* Enters the VT-x session.
6048	*
6049	* @returns VBox status code.
6050	* @param pVM Pointer to the VM.
6051	* @param pVCpu Pointer to the VMCPU.
6052	* @param pCpu Pointer to the CPU info struct.
6053	*/
6054	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6055	{
6056	AssertPtr(pVM);
6057	AssertPtr(pVCpu);
6058	Assert(pVM->hm.s.vmx.fSupported);
6059	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6060	NOREF(pCpu);
6061
6062	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6063
6064	/* Make sure we're in VMX root mode. */
6065	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6066	if (!(u32HostCR4 & X86_CR4_VMXE))
6067	{
6068	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6069	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6070	}
6071
6072	/* Load the active VMCS as the current one. */
6073	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6074	if (RT_FAILURE(rc))
6075	return rc;
6076
6077	/** @todo this will change with preemption hooks where can can VMRESUME as long
6078	* as we're no preempted. */
6079	pVCpu->hm.s.fResumeVM = false;
6080	return VINF_SUCCESS;
6081	}
6082
6083
6084	/**
6085	* Leaves the VT-x session.
6086	*
6087	* @returns VBox status code.
6088	* @param pVM Pointer to the VM.
6089	* @param pVCpu Pointer to the VMCPU.
6090	* @param pCtx Pointer to the guest-CPU context.
6091	*/
6092	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6093	{
6094	AssertPtr(pVCpu);
6095	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6096	NOREF(pVM);
6097	NOREF(pCtx);
6098
6099	/** @todo this will change with preemption hooks where we only VMCLEAR when
6100	* we are actually going to be preempted, not all the time like we
6101	* currently do. */
6102	/*
6103	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6104	* and mark the VMCS launch-state as "clear".
6105	*/
6106	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6107	return rc;
6108	}
6109
6110
6111	/**
6112	* Saves the host state in the VMCS host-state.
6113	* Sets up the VM-exit MSR-load area.
6114	*
6115	* The CPU state will be loaded from these fields on every successful VM-exit.
6116	*
6117	* @returns VBox status code.
6118	* @param pVM Pointer to the VM.
6119	* @param pVCpu Pointer to the VMCPU.
6120	*
6121	* @remarks No-long-jump zone!!!
6122	*/
6123	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6124	{
6125	AssertPtr(pVM);
6126	AssertPtr(pVCpu);
6127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6128
6129	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6130
6131	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6132	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6133	return VINF_SUCCESS;
6134
6135	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6136	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6137
6138	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6139	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6140
6141	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6142	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6143
6144	/* Reset the host-state-changed flag. */
6145	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6146	return rc;
6147	}
6148
6149
6150	/**
6151	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6152	* loaded from these fields on every successful VM-entry.
6153	*
6154	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6155	* Sets up the VM-entry controls.
6156	* Sets up the appropriate VMX non-root function to execute guest code based on
6157	* the guest CPU mode.
6158	*
6159	* @returns VBox status code.
6160	* @param pVM Pointer to the VM.
6161	* @param pVCpu Pointer to the VMCPU.
6162	* @param pCtx Pointer to the guest-CPU context.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6167	{
6168	AssertPtr(pVM);
6169	AssertPtr(pVCpu);
6170	AssertPtr(pCtx);
6171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6172
6173	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6174
6175	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6176
6177	/* Determine real-on-v86 mode. */
6178	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6179	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6180	&& CPUMIsGuestInRealModeEx(pCtx))
6181	{
6182	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6183	}
6184
6185	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6186	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6187
6188	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6189	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6190
6191	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6192	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6193
6194	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6195	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6196
6197	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6198	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6199
6200	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6201	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6202
6203	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6204	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6205
6206	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6207	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6208
6209	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6210	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6211
6212	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6213	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6214
6215	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6216	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6217	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6218
6219	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6220	return rc;
6221	}
6222
6223
6224	/**
6225	* Does the preparations before executing guest code in VT-x.
6226	*
6227	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6228	* recompiler. We must be cautious what we do here regarding committing
6229	* guest-state information into the the VMCS assuming we assuredly execute the
6230	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6231	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6232	* that the recompiler can (and should) use them when it resumes guest
6233	* execution. Otherwise such operations must be done when we can no longer
6234	* exit to ring-3.
6235	*
6236	* @returns VBox status code (informational status codes included).
6237	* @retval VINF_SUCCESS if we can proceed with running the guest.
6238	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6239	* into the guest.
6240	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6241	*
6242	* @param pVM Pointer to the VM.
6243	* @param pVCpu Pointer to the VMCPU.
6244	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6245	* out-of-sync. Make sure to update the required fields
6246	* before using them.
6247	* @param pVmxTransient Pointer to the VMX transient structure.
6248	*
6249	* @remarks Called with preemption disabled.
6250	*/
6251	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6252	{
6253	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6254
6255	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6256	PGMRZDynMapFlushAutoSet(pVCpu);
6257	#endif
6258
6259	/* Check force flag actions that might require us to go back to ring-3. */
6260	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6261	if (rc != VINF_SUCCESS)
6262	return rc;
6263
6264	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6265	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6266	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6267	{
6268	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6269	RTGCPHYS GCPhysApicBase;
6270	GCPhysApicBase = pMixedCtx->msrApicBase;
6271	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6272
6273	/* Unalias any existing mapping. */
6274	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6275	AssertRCReturn(rc, rc);
6276
6277	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6278	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6279	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6280	AssertRCReturn(rc, rc);
6281
6282	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6283	}
6284
6285	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6286	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6287	pVmxTransient->uEFlags = ASMIntDisableFlags();
6288	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6289	{
6290	ASMSetFlags(pVmxTransient->uEFlags);
6291	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6292	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6293	return VINF_EM_RAW_INTERRUPT;
6294	}
6295	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6296	#endif
6297
6298	/*
6299	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6300	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6301	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6302	*/
6303	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6304	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6305	AssertRCReturn(rc, rc);
6306	return rc;
6307	}
6308
6309
6310	/**
6311	* Prepares to run guest code in VT-x and we've committed to doing so. This
6312	* means there is no backing out to ring-3 or anywhere else at this
6313	* point.
6314	*
6315	* @param pVM Pointer to the VM.
6316	* @param pVCpu Pointer to the VMCPU.
6317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6318	* out-of-sync. Make sure to update the required fields
6319	* before using them.
6320	* @param pVmxTransient Pointer to the VMX transient structure.
6321	*
6322	* @remarks Called with preemption disabled.
6323	* @remarks No-long-jump zone!!!
6324	*/
6325	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6326	{
6327	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6328	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6329
6330	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6331	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6332	pVmxTransient->uEFlags = ASMIntDisableFlags();
6333	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6334	#endif
6335
6336	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6337	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6338	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6339	int rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6340	AssertRC(rc);
6341	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6342
6343	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6344	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6345	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6346
6347	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6348	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6349	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6350
6351	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6352	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6353	{
6354	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6355	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6356	}
6357
6358	/*
6359	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6360	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6361	*/
6362	if (pVM->hm.s.fTPRPatchingActive)
6363	{
6364	Assert(!CPUMIsGuestInLongMode(pVCpu));
6365
6366	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6367	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6368	AssertRC(rc);
6369
6370	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6371	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6372	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6373	see hmR0VmxLoadGuestApicState(). */
6374	}
6375
6376	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6377	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6378	to start executing. */
6379	}
6380
6381
6382	/**
6383	* Performs some essential restoration of state after running guest code in
6384	* VT-x.
6385	*
6386	* @param pVM Pointer to the VM.
6387	* @param pVCpu Pointer to the VMCPU.
6388	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6389	* out-of-sync. Make sure to update the required fields
6390	* before using them.
6391	* @param pVmxTransient Pointer to the VMX transient structure.
6392	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6393	*
6394	* @remarks Called with interrupts disabled.
6395	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6396	* unconditionally when it is safe to do so.
6397	*/
6398	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6399	{
6400	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6401	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6402
6403	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6404	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6405	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6406	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6407	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6408
6409	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6410	{
6411	/** @todo Find a way to fix hardcoding a guestimate. */
6412	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6413	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6414	}
6415
6416	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6417	Assert(!(ASMGetFlags() & X86_EFL_IF));
6418	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6419
6420	/* Restore the effects of TPR patching if any. */
6421	if (pVM->hm.s.fTPRPatchingActive)
6422	{
6423	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6424	AssertRC(rc);
6425	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6426	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6427	}
6428
6429	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6430	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6431
6432	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6433	uint32_t uExitReason;
6434	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6435	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6436	AssertRC(rc);
6437	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6438	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6439
6440	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6441	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6442
6443	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6444	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6445	{
6446	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6447	return;
6448	}
6449
6450	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6451	{
6452	/* Update the guest interruptibility-state from the VMCS. */
6453	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6454
6455	/*
6456	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6457	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6458	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6459	*/
6460	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6461	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6462	{
6463	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6464	AssertRC(rc);
6465	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6466	}
6467	}
6468	}
6469
6470
6471	/**
6472	* Runs the guest code using VT-x.
6473	*
6474	* @returns VBox status code.
6475	* @param pVM Pointer to the VM.
6476	* @param pVCpu Pointer to the VMCPU.
6477	* @param pCtx Pointer to the guest-CPU context.
6478	*
6479	* @remarks Called with preemption disabled.
6480	*/
6481	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6482	{
6483	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6484	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6485
6486	VMXTRANSIENT VmxTransient;
6487	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6488	int rc = VERR_INTERNAL_ERROR_5;
6489	unsigned cLoops = 0;
6490
6491	for (;; cLoops++)
6492	{
6493	Assert(!HMR0SuspendPending());
6494	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6495	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6496	(unsigned)RTMpCpuId(), cLoops));
6497
6498	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6499	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6500	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6501	if (rc != VINF_SUCCESS)
6502	break;
6503
6504	/*
6505	* No longjmps to ring-3 from this point on!!!
6506	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6507	* This also disables flushing of the R0-logger instance (if any).
6508	*/
6509	VMMRZCallRing3Disable(pVCpu);
6510	VMMRZCallRing3RemoveNotification(pVCpu);
6511	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6512
6513	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6514	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6515
6516	/*
6517	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6518	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6519	*/
6520	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6521	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6522	{
6523	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6524	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6525	return rc;
6526	}
6527
6528	/* Handle the VM-exit. */
6529	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6530	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6531	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6532	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6533	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6534	if (rc != VINF_SUCCESS)
6535	break;
6536	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6537	{
6538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6539	rc = VINF_EM_RAW_INTERRUPT;
6540	break;
6541	}
6542	}
6543
6544	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6545	if (rc == VERR_EM_INTERPRETER)
6546	rc = VINF_EM_RAW_EMULATE_INSTR;
6547	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6548	return rc;
6549	}
6550
6551	#ifdef DEBUG
6552	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6553	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6554	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6555	# define VMX_ASSERT_PREEMPT_CPUID() \
6556	do \
6557	{ \
6558	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6559	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6560	} while (0)
6561
6562	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6563	do { \
6564	AssertPtr(pVM); \
6565	AssertPtr(pVCpu); \
6566	AssertPtr(pMixedCtx); \
6567	AssertPtr(pVmxTransient); \
6568	Assert(pVmxTransient->fVMEntryFailed == false); \
6569	Assert(ASMIntAreEnabled() == true); \
6570	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6571	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6572	LogFunc(("\n")); \
6573	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6574	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6575	VMX_ASSERT_PREEMPT_CPUID(); \
6576	} while (0)
6577	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6578	do { \
6579	LogFunc(("\n")); \
6580	} while(0)
6581	#else /* Release builds */
6582	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6583	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6584	#endif
6585
6586
6587	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6588	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6589	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6590	/**
6591	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6592	*/
6593	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6594	{
6595	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6597	return VINF_SUCCESS;
6598	}
6599
6600
6601	/**
6602	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6603	*/
6604	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6605	{
6606	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6607	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6608	AssertRCReturn(rc, rc);
6609
6610	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6611	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6612	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6613
6614	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6615	return VINF_EM_RAW_INTERRUPT;
6616
6617	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6618	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6619	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6620	return VINF_SUCCESS;
6621	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6622	return rc;
6623
6624	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6625	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6626	switch (uIntrType)
6627	{
6628	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6629	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6630	/* no break */
6631	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6632	{
6633	switch (uVector)
6634	{
6635	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6636	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6637	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6638	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6639	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6640	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6641	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6642	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6643	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6644	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6645	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6646	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6647	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6648	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6649	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6650	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6651	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6652	#endif
6653	default:
6654	{
6655	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6656	AssertRCReturn(rc, rc);
6657
6658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6659	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6660	{
6661	Assert(pVM->hm.s.vmx.pRealModeTSS);
6662	Assert(PDMVmmDevHeapIsEnabled(pVM));
6663	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6664	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6665	AssertRCReturn(rc, rc);
6666	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6667	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6668	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6669	AssertRCReturn(rc, rc);
6670	}
6671	else
6672	{
6673	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6674	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6675	}
6676	break;
6677	}
6678	}
6679	break;
6680	}
6681
6682	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6683	default:
6684	{
6685	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6686	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6687	break;
6688	}
6689	}
6690	return rc;
6691	}
6692
6693
6694	/**
6695	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6696	*/
6697	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6698	{
6699	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6700
6701	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6702	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6703	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6704	AssertRCReturn(rc, rc);
6705
6706	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6708	return VINF_SUCCESS;
6709	}
6710
6711
6712	/**
6713	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6714	*/
6715	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6716	{
6717	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6718	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6719	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6720	}
6721
6722
6723	/**
6724	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6725	*/
6726	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6727	{
6728	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6729	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6730	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6731	AssertRCReturn(rc, rc);
6732
6733	pMixedCtx->rip += pVmxTransient->cbInstr;
6734	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6735
6736	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6737	return VINF_SUCCESS;
6738	}
6739
6740
6741	/**
6742	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6743	*/
6744	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6745	{
6746	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6747	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6748	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6749	AssertRCReturn(rc, rc);
6750
6751	pMixedCtx->rip += pVmxTransient->cbInstr;
6752	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6753
6754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6755	return VINF_SUCCESS;
6756	}
6757
6758
6759	/**
6760	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6761	*/
6762	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6763	{
6764	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6765	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6766	if (RT_LIKELY(rc == VINF_SUCCESS))
6767	{
6768	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6769	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6770	AssertRCReturn(rc, rc);
6771	Assert(pVmxTransient->cbInstr == 2);
6772
6773	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6774	pMixedCtx->rip += pVmxTransient->cbInstr;
6775	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6776	}
6777	else
6778	{
6779	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6780	rc = VERR_EM_INTERPRETER;
6781	}
6782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6783	return rc;
6784	}
6785
6786
6787	/**
6788	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6789	*/
6790	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6791	{
6792	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6793	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6794	AssertRCReturn(rc, rc);
6795
6796	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6797	return VINF_EM_RAW_EMULATE_INSTR;
6798
6799	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6800	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6801	}
6802
6803
6804	/**
6805	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6806	*/
6807	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6808	{
6809	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6810	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6811	AssertRCReturn(rc, rc);
6812
6813	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6814	if (RT_LIKELY(rc == VINF_SUCCESS))
6815	{
6816	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6817	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6818	AssertRCReturn(rc, rc);
6819	Assert(pVmxTransient->cbInstr == 2);
6820
6821	pMixedCtx->rip += pVmxTransient->cbInstr;
6822	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6823
6824	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6825	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6826	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
6827	}
6828	else
6829	{
6830	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6831	rc = VERR_EM_INTERPRETER;
6832	}
6833	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6834	return rc;
6835	}
6836
6837
6838	/**
6839	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6840	*/
6841	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6842	{
6843	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6844	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6845	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6846	AssertRCReturn(rc, rc);
6847
6848	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6849	if (RT_LIKELY(rc == VINF_SUCCESS))
6850	{
6851	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6852	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6853	AssertRCReturn(rc, rc);
6854	Assert(pVmxTransient->cbInstr == 3);
6855
6856	pMixedCtx->rip += pVmxTransient->cbInstr;
6857	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6858
6859	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6860	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6861	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
6862	}
6863	else
6864	{
6865	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6866	rc = VERR_EM_INTERPRETER;
6867	}
6868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6869	return rc;
6870	}
6871
6872
6873	/**
6874	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6875	*/
6876	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6877	{
6878	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6879	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6880	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6881	AssertRCReturn(rc, rc);
6882
6883	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6884	if (RT_LIKELY(rc == VINF_SUCCESS))
6885	{
6886	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6887	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6888	AssertRCReturn(rc, rc);
6889	Assert(pVmxTransient->cbInstr == 2);
6890
6891	pMixedCtx->rip += pVmxTransient->cbInstr;
6892	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6893	}
6894	else
6895	{
6896	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6897	rc = VERR_EM_INTERPRETER;
6898	}
6899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6900	return rc;
6901	}
6902
6903
6904	/**
6905	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6906	*/
6907	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6908	{
6909	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6910	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6911	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6912	AssertRCReturn(rc, rc);
6913
6914	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6915	rc = VBOXSTRICTRC_VAL(rc2);
6916	if (RT_LIKELY(rc == VINF_SUCCESS))
6917	{
6918	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6919	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6920	AssertRCReturn(rc, rc);
6921
6922	pMixedCtx->rip += pVmxTransient->cbInstr;
6923	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6924	}
6925	else
6926	{
6927	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6928	pVmxTransient->uExitQualification, rc));
6929	rc = VERR_EM_INTERPRETER;
6930	}
6931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6932	return rc;
6933	}
6934
6935
6936	/**
6937	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6938	*/
6939	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6940	{
6941	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6942	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6943	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6944	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6945	AssertRCReturn(rc, rc);
6946
6947	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6948	if (RT_LIKELY(rc == VINF_SUCCESS))
6949	{
6950	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6951	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6952	AssertRCReturn(rc, rc);
6953
6954	pMixedCtx->rip += pVmxTransient->cbInstr;
6955	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6956	}
6957	else
6958	{
6959	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6960	rc = VERR_EM_INTERPRETER;
6961	}
6962	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6963	return rc;
6964	}
6965
6966
6967	/**
6968	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
6969	*/
6970	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6971	{
6972	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6973	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6974	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6975	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6976	AssertRCReturn(rc, rc);
6977
6978	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6979	rc = VBOXSTRICTRC_VAL(rc2);
6980	if (RT_LIKELY( rc == VINF_SUCCESS
6981	\|\| rc == VINF_EM_HALT))
6982	{
6983	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6984	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6985	AssertRCReturn(rc3, rc3);
6986
6987	pMixedCtx->rip += pVmxTransient->cbInstr;
6988	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6989
6990	if ( rc == VINF_EM_HALT
6991	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
6992	{
6993	rc = VINF_SUCCESS;
6994	}
6995	}
6996	else
6997	{
6998	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
6999	rc = VERR_EM_INTERPRETER;
7000	}
7001	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7002	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7004	return rc;
7005	}
7006
7007
7008	/**
7009	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7010	*/
7011	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7012	{
7013	/*
7014	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7015	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7016	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7017	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7018	*/
7019	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7020	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7021	}
7022
7023
7024	/**
7025	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7026	*/
7027	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7028	{
7029	/*
7030	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7031	* root operation. If we get there there is something funny going on.
7032	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7033	*/
7034	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7035	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7036	}
7037
7038
7039	/**
7040	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7041	*/
7042	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7043	{
7044	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7045	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7046	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7047	}
7048
7049
7050	/**
7051	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7052	*/
7053	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7054	{
7055	/*
7056	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7057	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7058	* See Intel spec. 25.3 "Other Causes of VM-exits".
7059	*/
7060	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7061	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7062	}
7063
7064
7065	/**
7066	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7067	* VM-exit.
7068	*/
7069	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7070	{
7071	/*
7072	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7073	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7074	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7075	*/
7076	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7077	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7078	}
7079
7080
7081	/**
7082	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7083	* VM-exit.
7084	*/
7085	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7086	{
7087	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7088	return VINF_EM_RESET;
7089	}
7090
7091
7092	/**
7093	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7094	*/
7095	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7096	{
7097	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7098	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7099	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7100	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7101	AssertRCReturn(rc, rc);
7102
7103	pMixedCtx->rip++;
7104	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7105	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7106	rc = VINF_SUCCESS;
7107	else
7108	rc = VINF_EM_HALT;
7109
7110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7111	return rc;
7112	}
7113
7114
7115	/**
7116	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7117	*/
7118	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7119	{
7120	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7121	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7122	AssertRCReturn(rc, rc);
7123	return rc;
7124	}
7125
7126
7127	/**
7128	* VM-exit handler for expiry of the VMX preemption timer.
7129	*/
7130	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7131	{
7132	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7133
7134	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7135	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7136	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7137
7138	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7139	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7141	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7142	}
7143
7144
7145	/**
7146	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7147	*/
7148	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7149	{
7150	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7151	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7152	/** @todo check if XSETBV is supported by the recompiler. */
7153	return VERR_EM_INTERPRETER;
7154	}
7155
7156
7157	/**
7158	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7159	*/
7160	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7161	{
7162	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7163	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7164	/** @todo implement EMInterpretInvpcid() */
7165	return VERR_EM_INTERPRETER;
7166	}
7167
7168
7169	/**
7170	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7171	* Error VM-exit.
7172	*/
7173	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7174	{
7175	uint32_t uIntrState;
7176	RTHCUINTREG uHCReg;
7177	uint64_t u64Val;
7178
7179	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7180	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7181	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7182	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7183	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7184	AssertRCReturn(rc, rc);
7185
7186	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7187	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7188	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7189	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7190
7191	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7192	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7193	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7194	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7195	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7196	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7197	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7198	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7199	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7200	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7201	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7202	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7203
7204	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7205
7206	return VERR_VMX_INVALID_GUEST_STATE;
7207	}
7208
7209
7210	/**
7211	* VM-exit handler for VM-entry failure due to an MSR-load
7212	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7213	*/
7214	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7215	{
7216	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7217	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7218	}
7219
7220
7221	/**
7222	* VM-exit handler for VM-entry failure due to a machine-check event
7223	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7224	*/
7225	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7226	{
7227	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7228	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7229	}
7230
7231
7232	/**
7233	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7234	* theory.
7235	*/
7236	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7237	{
7238	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7239	pVM, pVCpu, pMixedCtx));
7240	return VERR_VMX_UNDEFINED_EXIT_CODE;
7241	}
7242
7243
7244	/**
7245	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7246	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7247	* Conditional VM-exit.
7248	*/
7249	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7250	{
7251	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7252	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7253	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7254	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7255	return VERR_EM_INTERPRETER;
7256	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7257	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7258	}
7259
7260
7261	/**
7262	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7263	*/
7264	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7265	{
7266	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7267	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7268	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7269	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7270	return VERR_EM_INTERPRETER;
7271	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7272	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7273	}
7274
7275
7276	/**
7277	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7278	*/
7279	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7280	{
7281	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7282	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7283	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7284	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7285	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7286	AssertRCReturn(rc, rc);
7287
7288	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7289	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7290	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7292
7293	/* Update RIP and continue guest execution. */
7294	if (RT_LIKELY(rc == VINF_SUCCESS))
7295	{
7296	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7297	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7298	AssertRCReturn(rc, rc);
7299
7300	Assert(pVmxTransient->cbInstr == 2);
7301	pMixedCtx->rip += pVmxTransient->cbInstr;
7302	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7303	}
7304	return rc;
7305	}
7306
7307
7308	/**
7309	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7310	*/
7311	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7312	{
7313	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7314	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7315	AssertRCReturn(rc, rc);
7316	Assert(pVmxTransient->cbInstr == 2);
7317
7318	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7319	if ( pVM->hm.s.fTPRPatchingActive
7320	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7321	{
7322	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7323	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7324	{
7325	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7326	AssertRC(rc);
7327	}
7328
7329	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7330	AssertRCReturn(rc, rc);
7331	pMixedCtx->rip += pVmxTransient->cbInstr;
7332	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7334	return VINF_SUCCESS;
7335	}
7336
7337	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7338	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7339	{
7340	switch (pMixedCtx->ecx)
7341	{
7342	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7343	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7344	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7345	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7346	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7347	}
7348	}
7349	#ifdef DEBUG
7350	else
7351	{
7352	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7353	switch (pMixedCtx->ecx)
7354	{
7355	case MSR_IA32_SYSENTER_CS:
7356	case MSR_IA32_SYSENTER_EIP:
7357	case MSR_IA32_SYSENTER_ESP:
7358	case MSR_K8_FS_BASE:
7359	case MSR_K8_GS_BASE:
7360	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7361	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7362	case MSR_K8_LSTAR:
7363	case MSR_K6_STAR:
7364	case MSR_K8_SF_MASK:
7365	case MSR_K8_TSC_AUX:
7366	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7367	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7368	}
7369	}
7370	#endif
7371
7372	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7373	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7374	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7375	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7376	AssertRCReturn(rc, rc);
7377
7378	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7379	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7381
7382	/* Update guest-state and continue execution. */
7383	if (RT_LIKELY(rc == VINF_SUCCESS))
7384	{
7385	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7386	AssertRCReturn(rc, rc);
7387
7388	pMixedCtx->rip += pVmxTransient->cbInstr;
7389	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7390
7391	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7392	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7393	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7394	{
7395	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7396	}
7397	}
7398	return rc;
7399	}
7400
7401
7402	/**
7403	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7404	*/
7405	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7406	{
7407	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7408	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7410	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7411	return VERR_EM_INTERPRETER;
7412	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7413	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7414	}
7415
7416
7417	/**
7418	* VM-exit handler for when the TPR value is lowered below the specified
7419	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7420	*/
7421	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7422	{
7423	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7424	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7425
7426	/*
7427	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7428	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7429	* resume guest execution.
7430	*/
7431	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7432	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7433	return VINF_SUCCESS;
7434	}
7435
7436
7437	/**
7438	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7439	* VM-exit.
7440	*
7441	* @retval VINF_SUCCESS when guest execution can continue.
7442	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7443	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7444	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7445	* recompiler.
7446	*/
7447	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7448	{
7449	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7450	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7451	AssertRCReturn(rc, rc);
7452
7453	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7454	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7455	switch (uAccessType)
7456	{
7457	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7458	{
7459	#if 0
7460	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7461	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7462	#else
7463	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7464	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7465	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7466	#endif
7467	AssertRCReturn(rc, rc);
7468
7469	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7470	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7471	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7472	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7473
7474	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7475	{
7476	case 0: /* CR0 */
7477	Log(("CR0 write rc=%d\n", rc));
7478	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7479	break;
7480	case 2: /* CR2 */
7481	Log(("CR2 write rc=%d\n", rc));
7482	break;
7483	case 3: /* CR3 */
7484	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7485	Log(("CR3 write rc=%d\n", rc));
7486	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7487	break;
7488	case 4: /* CR4 */
7489	Log(("CR4 write rc=%d\n", rc));
7490	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7491	break;
7492	case 8: /* CR8 */
7493	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7494	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7495	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7496	break;
7497	default:
7498	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7499	break;
7500	}
7501
7502	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7503	break;
7504	}
7505
7506	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7507	{
7508	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7509	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7510	AssertRCReturn(rc, rc);
7511	Assert( !pVM->hm.s.fNestedPaging
7512	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7513	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7514
7515	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7516	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7517	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7518
7519	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7520	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7521	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7522	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7524	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7525	break;
7526	}
7527
7528	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7529	{
7530	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7531	rc \|= EMInterpretCLTS(pVM, pVCpu);
7532	AssertRCReturn(rc, rc);
7533	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7535	Log(("CRX CLTS write rc=%d\n", rc));
7536	break;
7537	}
7538
7539	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7540	{
7541	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7542	AssertRCReturn(rc, rc);
7543	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7544	if (RT_LIKELY(rc == VINF_SUCCESS))
7545	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7547	Log(("CRX LMSW write rc=%d\n", rc));
7548	break;
7549	}
7550
7551	default:
7552	{
7553	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7554	return VERR_VMX_UNEXPECTED_EXCEPTION;
7555	}
7556	}
7557
7558	/* Validate possible error codes. */
7559	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7560	if (RT_SUCCESS(rc))
7561	{
7562	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7563	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7564	AssertRCReturn(rc2, rc2);
7565	pMixedCtx->rip += pVmxTransient->cbInstr;
7566	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7567	}
7568
7569	return rc;
7570	}
7571
7572
7573	/**
7574	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7575	* VM-exit.
7576	*/
7577	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7578	{
7579	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7580
7581	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7582	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7583	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7584	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7585	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7586	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7587	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7588	AssertRCReturn(rc, rc);
7589
7590	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7591	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7592	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7593	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7594	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7595	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7596	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7597
7598	/* I/O operation lookup arrays. */
7599	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7600	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7601
7602	const uint32_t cbSize = s_aIOSize[uIOWidth];
7603	const uint32_t cbInstr = pVmxTransient->cbInstr;
7604	if (fIOString)
7605	{
7606	/* INS/OUTS - I/O String instruction. */
7607	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7608	/** @todo for now manually disassemble later optimize by getting the fields from
7609	* the VMCS. */
7610	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7611	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7612	* segment prefix info. */
7613	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7614	if (RT_SUCCESS(rc))
7615	{
7616	if (fIOWrite)
7617	{
7618	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7619	(DISCPUMODE)pDis->uAddrMode, cbSize);
7620	rc = VBOXSTRICTRC_VAL(rc2);
7621	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7622	}
7623	else
7624	{
7625	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7626	(DISCPUMODE)pDis->uAddrMode, cbSize);
7627	rc = VBOXSTRICTRC_VAL(rc2);
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7629	}
7630	}
7631	else
7632	{
7633	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7634	rc = VINF_EM_RAW_EMULATE_INSTR;
7635	}
7636	}
7637	else
7638	{
7639	/* IN/OUT - I/O instruction. */
7640	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7641	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7642	if (fIOWrite)
7643	{
7644	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7645	rc = VBOXSTRICTRC_VAL(rc2);
7646	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7647	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7649	}
7650	else
7651	{
7652	uint32_t u32Result = 0;
7653	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7654	rc = VBOXSTRICTRC_VAL(rc2);
7655	if (IOM_SUCCESS(rc))
7656	{
7657	/* Save result of I/O IN instr. in AL/AX/EAX. */
7658	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7659	}
7660	else if (rc == VINF_IOM_R3_IOPORT_READ)
7661	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7663	}
7664	}
7665
7666	if (IOM_SUCCESS(rc))
7667	{
7668	pMixedCtx->rip += cbInstr;
7669	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7670	if (RT_LIKELY(rc == VINF_SUCCESS))
7671	{
7672	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7673	AssertRCReturn(rc, rc);
7674
7675	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7676	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7677	{
7678	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7679	for (unsigned i = 0; i < 4; i++)
7680	{
7681	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7682	if ( ( uIOPort >= pMixedCtx->dr[i]
7683	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7684	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7685	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7686	{
7687	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7688	uint64_t uDR6 = ASMGetDR6();
7689
7690	/* Clear all breakpoint status flags and set the one we just hit. */
7691	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7692	uDR6 \|= (uint64_t)RT_BIT(i);
7693
7694	/*
7695	* Note: AMD64 Architecture Programmer's Manual 13.1:
7696	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7697	* be cleared by software after the contents have been read.
7698	*/
7699	ASMSetDR6(uDR6);
7700
7701	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7702	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7703
7704	/* Paranoia. */
7705	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7706	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7707	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7708
7709	/* Resync DR7 */
7710	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7711	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7712
7713	/* Inject #DB and get on with guest execution. */
7714	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7715	AssertRCReturn(rc, rc);
7716	break;
7717	}
7718	}
7719	}
7720	}
7721	}
7722
7723	#ifdef DEBUG
7724	if (rc == VINF_IOM_R3_IOPORT_READ)
7725	Assert(!fIOWrite);
7726	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7727	Assert(fIOWrite);
7728	else
7729	{
7730	AssertMsg( RT_FAILURE(rc)
7731	\|\| rc == VINF_SUCCESS
7732	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7733	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7734	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7735	}
7736	#endif
7737
7738	return rc;
7739	}
7740
7741
7742	/**
7743	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7744	* VM-exit.
7745	*/
7746	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7747	{
7748	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7749
7750	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7751	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7752	AssertRCReturn(rc, rc);
7753	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7754	{
7755	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7756	AssertRCReturn(rc, rc);
7757	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7758	{
7759	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7760	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7761	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7762	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7763	{
7764	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7765	pVCpu->hm.s.Event.fPending = true;
7766	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7767	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7768	AssertRCReturn(rc, rc);
7769	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7770	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7771	else
7772	pVCpu->hm.s.Event.u32ErrCode = 0;
7773	}
7774	}
7775	}
7776	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7777	* emulation. */
7778	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7779	return VERR_EM_INTERPRETER;
7780	}
7781
7782
7783	/**
7784	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7785	*/
7786	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7787	{
7788	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7789	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7790	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7791	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7792	AssertRCReturn(rc, rc);
7793	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7794	return VINF_EM_DBG_STOP;
7795	}
7796
7797
7798	/**
7799	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7800	*/
7801	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7802	{
7803	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7804	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7805
7806	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7807	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7808	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7809	return VINF_SUCCESS;
7810	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7811	return rc;
7812
7813	#if 0
7814	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7815	* just sync the whole thing. */
7816	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7817	#else
7818	/* Aggressive state sync. for now. */
7819	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7820	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7821	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7822	#endif
7823	AssertRCReturn(rc, rc);
7824
7825	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7826	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7827	switch (uAccessType)
7828	{
7829	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7830	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7831	{
7832	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7833	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7834	{
7835	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7836	}
7837
7838	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7839	GCPhys &= PAGE_BASE_GC_MASK;
7840	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7841	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7842	CPUMCTX2CORE(pMixedCtx), GCPhys);
7843	rc = VBOXSTRICTRC_VAL(rc2);
7844	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7845	if ( rc == VINF_SUCCESS
7846	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7847	\|\| rc == VERR_PAGE_NOT_PRESENT)
7848	{
7849	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7850	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7851	rc = VINF_SUCCESS;
7852	}
7853	break;
7854	}
7855
7856	default:
7857	rc = VINF_EM_RAW_EMULATE_INSTR;
7858	break;
7859	}
7860
7861	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7862	return rc;
7863	}
7864
7865
7866	/**
7867	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7868	* VM-exit.
7869	*/
7870	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7871	{
7872	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7873
7874	/* We should -not- get this VM-exit if the guest is debugging. */
7875	if (CPUMIsGuestDebugStateActive(pVCpu))
7876	{
7877	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7878	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7879	}
7880
7881	int rc = VERR_INTERNAL_ERROR_5;
7882	if ( !DBGFIsStepping(pVCpu)
7883	&& !CPUMIsHyperDebugStateActive(pVCpu))
7884	{
7885	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7886
7887	/* Don't intercept MOV DRx. */
7888	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7889	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7890	AssertRCReturn(rc, rc);
7891
7892	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7893	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7894	AssertRCReturn(rc, rc);
7895	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7896
7897	#ifdef VBOX_WITH_STATISTICS
7898	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7899	AssertRCReturn(rc, rc);
7900	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7902	else
7903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7904	#endif
7905	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7906	return VINF_SUCCESS;
7907	}
7908
7909	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7910	* time and restore DRx registers afterwards */
7911	/*
7912	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7913	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7914	*/
7915	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7916	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7917	AssertRCReturn(rc, rc);
7918
7919	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7920	{
7921	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7922	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7923	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7924	if (RT_SUCCESS(rc))
7925	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7926	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7927	}
7928	else
7929	{
7930	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7931	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7932	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7934	}
7935
7936	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7937	if (RT_SUCCESS(rc))
7938	{
7939	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7940	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7941	AssertRCReturn(rc2, rc2);
7942	pMixedCtx->rip += pVmxTransient->cbInstr;
7943	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7944	}
7945	return rc;
7946	}
7947
7948
7949	/**
7950	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7951	* Conditional VM-exit.
7952	*/
7953	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7954	{
7955	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7956	Assert(pVM->hm.s.fNestedPaging);
7957
7958	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7959	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7960	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7961	return VINF_SUCCESS;
7962	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7963	return rc;
7964
7965	RTGCPHYS GCPhys = 0;
7966	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
7967	AssertRCReturn(rc, rc);
7968
7969	#if 0
7970	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
7971	#else
7972	/* Aggressive state sync. for now. */
7973	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7974	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7975	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7976	#endif
7977	AssertRCReturn(rc, rc);
7978
7979	/*
7980	* If we succeed, resume guest execution.
7981	* If we fail in interpreting the instruction because we couldn't get the guest physical address
7982	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
7983	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
7984	* weird case. See @bugref{6043}.
7985	*/
7986	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
7987	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
7988	rc = VBOXSTRICTRC_VAL(rc2);
7989	if ( rc == VINF_SUCCESS
7990	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7991	\|\| rc == VERR_PAGE_NOT_PRESENT)
7992	{
7993	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7994	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7995	return VINF_SUCCESS;
7996	}
7997	return rc;
7998	}
7999
8000
8001	/**
8002	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8003	* VM-exit.
8004	*/
8005	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8006	{
8007	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8008	Assert(pVM->hm.s.fNestedPaging);
8009
8010	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8011	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8012	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8013	return VINF_SUCCESS;
8014	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8015	return rc;
8016
8017	RTGCPHYS GCPhys = 0;
8018	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8019	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8020	#if 0
8021	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8022	#else
8023	/* Aggressive state sync. for now. */
8024	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8025	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8026	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8027	#endif
8028	AssertRCReturn(rc, rc);
8029
8030	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8031	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8032
8033	RTGCUINT uErrorCode = 0;
8034	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8035	uErrorCode \|= X86_TRAP_PF_ID;
8036	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8037	uErrorCode \|= X86_TRAP_PF_RW;
8038	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8039	uErrorCode \|= X86_TRAP_PF_P;
8040
8041	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8042	TRPMSetErrorCode(pVCpu, uErrorCode);
8043	TRPMSetFaultAddress(pVCpu, GCPhys);
8044
8045	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8046	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8047
8048	/* Handle the pagefault trap for the nested shadow table. */
8049	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8050	TRPMResetTrap(pVCpu);
8051
8052	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8053	if ( rc == VINF_SUCCESS
8054	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8055	\|\| rc == VERR_PAGE_NOT_PRESENT)
8056	{
8057	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8059	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8060	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8061	return VINF_SUCCESS;
8062	}
8063
8064	Log(("EPT return to ring-3 rc=%d\n"));
8065
8066	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8067	/** @todo Shouldn't we update TRPM here? */
8068	return rc;
8069	}
8070
8071
8072	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8073	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8074	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8075	/**
8076	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8077	*/
8078	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8079	{
8080	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8081
8082	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8083	AssertRCReturn(rc, rc);
8084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8085
8086	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8087	{
8088	/* Old-style FPU error reporting needs some extra work. */
8089	/** @todo don't fall back to the recompiler, but do it manually. */
8090	return VERR_EM_INTERPRETER;
8091	}
8092	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8093	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8094	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8095	AssertRCReturn(rc, rc);
8096	return rc;
8097	}
8098
8099
8100	/**
8101	* VM-exit exception handler for #BP (Breakpoint exception).
8102	*/
8103	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8104	{
8105	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8106
8107	/** @todo Try optimize this by not saving the entire guest state unless
8108	* really needed. */
8109	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8110	AssertRCReturn(rc, rc);
8111	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8112
8113	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8114	if (rc == VINF_EM_RAW_GUEST_TRAP)
8115	{
8116	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8117	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8118	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8119	AssertRCReturn(rc, rc);
8120
8121	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8122	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8123	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8124	AssertRCReturn(rc, rc);
8125	}
8126
8127	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8128	return rc;
8129	}
8130
8131
8132	/**
8133	* VM-exit exception handler for #DB (Debug exception).
8134	*/
8135	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8136	{
8137	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8138
8139	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8140	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8141	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8142	AssertRCReturn(rc, rc);
8143
8144	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8145	uint64_t uDR6 = X86_DR6_INIT_VAL;
8146	uDR6 \|= (pVmxTransient->uExitQualification
8147	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8148	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8149	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8150	if (rc == VINF_EM_RAW_GUEST_TRAP)
8151	{
8152	/** @todo revisit this. */
8153	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8154	pMixedCtx->dr[6] = uDR6;
8155
8156	if (CPUMIsGuestDebugStateActive(pVCpu))
8157	ASMSetDR6(pMixedCtx->dr[6]);
8158
8159	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8160	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8161
8162	/* Paranoia. */
8163	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8164	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8165	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8166
8167	/* Resync DR7. */
8168	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8169
8170	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8171	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8172	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8173	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8174	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8175	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8176	AssertRCReturn(rc,rc);
8177	return rc;
8178	}
8179	/* Return to ring 3 to deal with the debug exit code. */
8180	return rc;
8181	}
8182
8183
8184	/**
8185	* VM-exit exception handler for #NM (Device-not-available exception: floating
8186	* point exception).
8187	*/
8188	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8189	{
8190	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8191
8192	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8193	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8194	#endif
8195
8196	/* We require CR0 and EFER. EFER is always up-to-date. */
8197	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8198	AssertRCReturn(rc, rc);
8199
8200	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8201	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8202	if (rc == VINF_SUCCESS)
8203	{
8204	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8205	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8207	return VINF_SUCCESS;
8208	}
8209
8210	/* Forward #NM to the guest. */
8211	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8212	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8213	AssertRCReturn(rc, rc);
8214	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8215	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8216	pVmxTransient->cbInstr, 0 /* error code */);
8217	AssertRCReturn(rc, rc);
8218	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8219	return rc;
8220	}
8221
8222
8223	/**
8224	* VM-exit exception handler for #GP (General-protection exception).
8225	*
8226	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8227	*/
8228	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8229	{
8230	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8231
8232	int rc = VERR_INTERNAL_ERROR_5;
8233
8234	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8235	{
8236	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8237	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8238	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8239	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8240	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8241	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8242	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8243	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8244	AssertRCReturn(rc, rc);
8245	return rc;
8246	#else
8247	/* We don't intercept #GP. */
8248	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8249	return VERR_VMX_UNEXPECTED_EXCEPTION;
8250	#endif
8251	}
8252
8253	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8254	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8255
8256	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8257	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8258	AssertRCReturn(rc, rc);
8259
8260	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8261	unsigned int cbOp = 0;
8262	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8263	if (RT_SUCCESS(rc))
8264	{
8265	rc = VINF_SUCCESS;
8266	Assert(cbOp == pDis->cbInstr);
8267	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8268	switch (pDis->pCurInstr->uOpcode)
8269	{
8270	case OP_CLI:
8271	pMixedCtx->eflags.Bits.u1IF = 0;
8272	pMixedCtx->rip += pDis->cbInstr;
8273	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8275	break;
8276
8277	case OP_STI:
8278	pMixedCtx->eflags.Bits.u1IF = 1;
8279	pMixedCtx->rip += pDis->cbInstr;
8280	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8281	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8282	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8284	break;
8285
8286	case OP_HLT:
8287	rc = VINF_EM_HALT;
8288	pMixedCtx->rip += pDis->cbInstr;
8289	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8290	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8291	break;
8292
8293	case OP_POPF:
8294	{
8295	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8296	size_t cbParm = 0;
8297	uint32_t uMask = 0;
8298	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8299	{
8300	cbParm = 4;
8301	uMask = 0xffffffff;
8302	}
8303	else
8304	{
8305	cbParm = 2;
8306	uMask = 0xffff;
8307	}
8308
8309	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8310	RTGCPTR GCPtrStack = 0;
8311	X86EFLAGS uEflags;
8312	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8313	&GCPtrStack);
8314	if (RT_SUCCESS(rc))
8315	{
8316	Assert(sizeof(uEflags.u32) >= cbParm);
8317	uEflags.u32 = 0;
8318	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8319	}
8320	if (RT_FAILURE(rc))
8321	{
8322	rc = VERR_EM_INTERPRETER;
8323	break;
8324	}
8325	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8326	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8327	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8328	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8329	pMixedCtx->eflags.Bits.u1RF = 0;
8330	pMixedCtx->esp += cbParm;
8331	pMixedCtx->esp &= uMask;
8332	pMixedCtx->rip += pDis->cbInstr;
8333	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8334	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8335	break;
8336	}
8337
8338	case OP_PUSHF:
8339	{
8340	size_t cbParm = 0;
8341	uint32_t uMask = 0;
8342	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8343	{
8344	cbParm = 4;
8345	uMask = 0xffffffff;
8346	}
8347	else
8348	{
8349	cbParm = 2;
8350	uMask = 0xffff;
8351	}
8352
8353	/* Get the stack pointer & push the contents of eflags onto the stack. */
8354	RTGCPTR GCPtrStack = 0;
8355	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8356	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8357	if (RT_FAILURE(rc))
8358	{
8359	rc = VERR_EM_INTERPRETER;
8360	break;
8361	}
8362	X86EFLAGS uEflags;
8363	uEflags = pMixedCtx->eflags;
8364	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8365	uEflags.Bits.u1RF = 0;
8366	uEflags.Bits.u1VM = 0;
8367
8368	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8369	if (RT_FAILURE(rc))
8370	{
8371	rc = VERR_EM_INTERPRETER;
8372	break;
8373	}
8374	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8375	pMixedCtx->esp -= cbParm;
8376	pMixedCtx->esp &= uMask;
8377	pMixedCtx->rip += pDis->cbInstr;
8378	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8380	break;
8381	}
8382
8383	case OP_IRET:
8384	{
8385	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8386	* instruction reference. */
8387	RTGCPTR GCPtrStack = 0;
8388	uint32_t uMask = 0xffff;
8389	uint16_t aIretFrame[3];
8390	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8391	{
8392	rc = VERR_EM_INTERPRETER;
8393	break;
8394	}
8395	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8396	&GCPtrStack);
8397	if (RT_SUCCESS(rc))
8398	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8399	if (RT_FAILURE(rc))
8400	{
8401	rc = VERR_EM_INTERPRETER;
8402	break;
8403	}
8404	pMixedCtx->eip = 0;
8405	pMixedCtx->ip = aIretFrame[0];
8406	pMixedCtx->cs.Sel = aIretFrame[1];
8407	pMixedCtx->cs.ValidSel = aIretFrame[1];
8408	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8409	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8410	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8411	pMixedCtx->sp += sizeof(aIretFrame);
8412	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8413	\| HM_CHANGED_GUEST_RFLAGS;
8414	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8415	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8416	break;
8417	}
8418
8419	case OP_INT:
8420	{
8421	uint16_t uVector = pDis->Param1.uValue & 0xff;
8422	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8423	AssertRCReturn(rc, rc);
8424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8425	break;
8426	}
8427
8428	case OP_INTO:
8429	{
8430	if (pMixedCtx->eflags.Bits.u1OF)
8431	{
8432	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8434	}
8435	break;
8436	}
8437
8438	default:
8439	{
8440	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8441	EMCODETYPE_SUPERVISOR);
8442	rc = VBOXSTRICTRC_VAL(rc2);
8443	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8444	Log2(("#GP rc=%Rrc\n", rc));
8445	break;
8446	}
8447	}
8448	}
8449	else
8450	rc = VERR_EM_INTERPRETER;
8451
8452	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8453	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8454	("#GP Unexpected rc=%Rrc\n", rc));
8455	return rc;
8456	}
8457
8458
8459	/**
8460	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8461	* the exception reported in the VMX transient structure back into the VM.
8462	*
8463	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8464	* VMX transient structure to be up-to-date.
8465	*/
8466	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8467	{
8468	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8469
8470	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8471	hmR0VmxCheckExitDueToEventDelivery(). */
8472	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8473	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8474	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8475	AssertRCReturn(rc, rc);
8476	return rc;
8477	}
8478
8479
8480	/**
8481	* VM-exit exception handler for #PF (Page-fault exception).
8482	*/
8483	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8484	{
8485	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8486
8487	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8488	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8489	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8490	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8491	AssertRCReturn(rc, rc);
8492
8493	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8494	if (pVM->hm.s.fNestedPaging)
8495	{
8496	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8497	{
8498	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8499	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8500	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8501	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8502	AssertRCReturn(rc, rc);
8503	}
8504	else
8505	{
8506	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8507	Assert(!pVCpu->hm.s.Event.fPending);
8508	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8509	}
8510	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8511	return rc;
8512	}
8513	#else
8514	Assert(!pVM->hm.s.fNestedPaging);
8515	#endif
8516
8517	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8518	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8519	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8520	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8521	AssertRCReturn(rc, rc);
8522	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8523	if ( pVM->hm.s.fTRPPatchingAllowed
8524	&& pVM->hm.s.pGuestPatchMem
8525	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8526	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8527	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8528	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8529	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8530	{
8531	RTGCPHYS GCPhys;
8532	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8533	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8534	if ( rc == VINF_SUCCESS
8535	&& GCPhys == GCPhysApicBase)
8536	{
8537	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8538	AssertRCReturn(rc, rc);
8539
8540	/* Only attempt to patch the instruction once. */
8541	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8542	if (!pPatch)
8543	return VINF_EM_HM_PATCH_TPR_INSTR;
8544	}
8545	}
8546	#endif
8547
8548	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8549	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8550	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8551
8552	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8553	AssertRCReturn(rc, rc);
8554
8555	/* Forward it to the trap handler first. */
8556	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8557	(RTGCPTR)pVmxTransient->uExitQualification);
8558
8559	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8560	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8561
8562	if (rc == VINF_SUCCESS)
8563	{
8564	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8565	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8566	* memory? We don't update the whole state here... */
8567	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8568	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8569
8570	TRPMResetTrap(pVCpu);
8571	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8572	return rc;
8573	}
8574	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8575	{
8576	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8577	{
8578	/* It's a guest page fault and needs to be reflected to the guest. */
8579	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8580	TRPMResetTrap(pVCpu);
8581	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8582	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8583	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8584	pVmxTransient->cbInstr, uGstErrorCode);
8585	AssertRCReturn(rc, rc);
8586	}
8587	else
8588	{
8589	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8590	Assert(!pVCpu->hm.s.Event.fPending);
8591	TRPMResetTrap(pVCpu);
8592	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8593	}
8594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8595	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8596	return rc;
8597	}
8598
8599	TRPMResetTrap(pVCpu);
8600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8601	return rc;
8602	}
8603

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45409

Download in other formats: