HMVMXR0.cpp@ 45413

Last change on this file since 45413 was 45413, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 360.1 KB

Line
1	/* $Id: HMVMXR0.cpp 45413 2013-04-08 20:15:01Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
76	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
77	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
79	\| VMX_UPDATED_GUEST_RIP \
80	\| VMX_UPDATED_GUEST_RSP \
81	\| VMX_UPDATED_GUEST_RFLAGS \
82	\| VMX_UPDATED_GUEST_CR0 \
83	\| VMX_UPDATED_GUEST_CR3 \
84	\| VMX_UPDATED_GUEST_CR4 \
85	\| VMX_UPDATED_GUEST_GDTR \
86	\| VMX_UPDATED_GUEST_IDTR \
87	\| VMX_UPDATED_GUEST_LDTR \
88	\| VMX_UPDATED_GUEST_TR \
89	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
90	\| VMX_UPDATED_GUEST_DEBUG \
91	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
92	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
96	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
97	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
98	\| VMX_UPDATED_GUEST_APIC_STATE)
99
100	/**
101	* Flags to skip redundant reads of some common VMCS fields.
102	*/
103	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
104	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
105	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
106	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
107	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
108	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
109
110	/*
111	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
112	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
113	* we have Nested Paging support.
114	*/
115	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
116	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
117	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
118	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
119	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
120	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
121	\| RT_BIT(X86_XCPT_XF))
122
123	/* Maximum VM-instruction error number. */
124	#define VMX_INSTR_ERROR_MAX 28
125
126	/*******************************************************************************
127	* Structures and Typedefs *
128	*******************************************************************************/
129	/**
130	* A state structure for holding miscellaneous information across
131	* VMX non-root operation and restored after the transition.
132	*/
133	typedef struct VMXTRANSIENT
134	{
135	/** The host's rflags/eflags. */
136	RTCCUINTREG uEFlags;
137	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
138	uint64_t u64LStarMsr;
139	/** The guest's TPR value used for TPR shadowing. */
140	uint8_t u8GuestTpr;
141
142	/** The basic VM-exit reason. */
143	uint16_t uExitReason;
144	/** The VM-exit exit qualification. */
145	RTGCUINTPTR uExitQualification;
146	/** The VM-exit interruption error code. */
147	uint32_t uExitIntrErrorCode;
148
149	/** The VM-exit interruption-information field. */
150	uint32_t uExitIntrInfo;
151	/** Whether the VM-entry failed or not. */
152	bool fVMEntryFailed;
153	/** The VM-exit instruction-length field. */
154	uint32_t cbInstr;
155
156	/** The VM-entry interruption-information field. */
157	uint32_t uEntryIntrInfo;
158	/** The VM-entry exception error code field. */
159	uint32_t uEntryXcptErrorCode;
160	/** The VM-entry instruction length field. */
161	uint32_t cbEntryInstr;
162
163	/** IDT-vectoring information field. */
164	uint32_t uIdtVectoringInfo;
165	/** IDT-vectoring error code. */
166	uint32_t uIdtVectoringErrorCode;
167
168	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
169	uint32_t fVmcsFieldsRead;
170	/** Whether TSC-offsetting should be setup before VM-entry. */
171	bool fUpdateTscOffsettingAndPreemptTimer;
172	/** Whether the VM-exit was caused by a page-fault during delivery of a
173	* contributary exception or a page-fault. */
174	bool fVectoringPF;
175	} VMXTRANSIENT, *PVMXTRANSIENT;
176
177	/**
178	* MSR-bitmap read permissions.
179	*/
180	typedef enum VMXMSREXITREAD
181	{
182	/** Reading this MSR causes a VM-exit. */
183	VMXMSREXIT_INTERCEPT_READ = 0xb,
184	/** Reading this MSR does not cause a VM-exit. */
185	VMXMSREXIT_PASSTHRU_READ
186	} VMXMSREXITREAD;
187
188	/**
189	* MSR-bitmap write permissions.
190	*/
191	typedef enum VMXMSREXITWRITE
192	{
193	/** Writing to this MSR causes a VM-exit. */
194	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
195	/** Writing to this MSR does not cause a VM-exit. */
196	VMXMSREXIT_PASSTHRU_WRITE
197	} VMXMSREXITWRITE;
198
199	/*******************************************************************************
200	* Internal Functions *
201	*******************************************************************************/
202	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
203	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
204	uint32_t cbInstr, uint32_t u32ErrCode);
205
206	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
207	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
208	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257
258	/*******************************************************************************
259	* Global Variables *
260	*******************************************************************************/
261	/** @todo Move this to hm_vmx.h. */
262	/**
263	* VM-exit handler.
264	*
265	* @returns VBox status code.
266	* @param pVM Pointer to the VM.
267	* @param pVCpu Pointer to the VMCPU.
268	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
269	* out-of-sync. Make sure to update the required
270	* fields before using them.
271	* @param pVmxTransient Pointer to the VMX-transient structure.
272	*/
273	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	/** Pointer to VM-exit handler. */
275	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
276
277	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
278	{
279	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
280	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
281	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
282	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
283	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
284	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
285	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
286	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
287	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
288	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
289	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
290	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
291	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
292	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
293	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
294	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
295	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
296	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
297	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
298	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
299	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
300	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
301	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
302	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
303	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
304	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
305	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
306	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
307	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
308	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
309	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
310	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
311	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
312	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
313	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
314	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
315	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
316	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
317	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
318	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
319	/* 40 UNDEFINED */ hmR0VmxExitPause,
320	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
321	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
322	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
323	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
324	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
325	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
326	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
327	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
328	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
329	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
330	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
331	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
332	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
333	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
334	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
335	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
336	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
337	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
338	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
339	};
340
341	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
342	{
343	/* 0 */ "(Not Used)",
344	/* 1 */ "VMCALL executed in VMX root operation.",
345	/* 2 */ "VMCLEAR with invalid physical address.",
346	/* 3 */ "VMCLEAR with VMXON pointer.",
347	/* 4 */ "VMLAUNCH with non-clear VMCS.",
348	/* 5 */ "VMRESUME with non-launched VMCS.",
349	/* 6 */ "VMRESUME after VMXOFF",
350	/* 7 */ "VM entry with invalid control fields.",
351	/* 8 */ "VM entry with invalid host state fields.",
352	/* 9 */ "VMPTRLD with invalid physical address.",
353	/* 10 */ "VMPTRLD with VMXON pointer.",
354	/* 11 */ "VMPTRLD with incorrect revision identifier.",
355	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
356	/* 13 */ "VMWRITE to read-only VMCS component.",
357	/* 14 */ "(Not Used)",
358	/* 15 */ "VMXON executed in VMX root operation.",
359	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
360	/* 17 */ "VM entry with non-launched executing VMCS.",
361	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
362	/* 19 */ "VMCALL with non-clear VMCS.",
363	/* 20 */ "VMCALL with invalid VM-exit control fields.",
364	/* 21 */ "(Not Used)",
365	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
366	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
367	/* 24 */ "VMCALL with invalid SMM-monitor features.",
368	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
369	/* 26 */ "VM entry with events blocked by MOV SS.",
370	/* 27 */ "(Not Used)",
371	/* 28 */ "Invalid operand to INVEPT/INVVPID."
372	};
373
374
375	/**
376	* Updates the VM's last error record. If there was a VMX instruction error,
377	* reads the error data from the VMCS and updates VCPU's last error record as
378	* well.
379	*
380	* @param pVM Pointer to the VM.
381	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
382	* VERR_VMX_GENERIC).
383	* @param rc The error code.
384	*/
385	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
386	{
387	AssertPtr(pVM);
388	if (rc == VERR_VMX_GENERIC)
389	{
390	AssertPtrReturnVoid(pVCpu);
391	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
392	}
393	pVM->hm.s.lLastError = rc;
394	}
395
396
397	/**
398	* Reads the VM-entry interruption-information field from the VMCS into the VMX
399	* transient structure.
400	*
401	* @returns VBox status code.
402	* @param pVmxTransient Pointer to the VMX transient structure.
403	*
404	* @remarks No-long-jump zone!!!
405	*/
406	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
407	{
408	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
409	AssertRCReturn(rc, rc);
410	return VINF_SUCCESS;
411	}
412
413
414	/**
415	* Reads the VM-entry exception error code field from the VMCS into
416	* the VMX transient structure.
417	*
418	* @returns VBox status code.
419	* @param pVmxTransient Pointer to the VMX transient structure.
420	*
421	* @remarks No-long-jump zone!!!
422	*/
423	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
424	{
425	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
426	AssertRCReturn(rc, rc);
427	return VINF_SUCCESS;
428	}
429
430
431	/**
432	* Reads the VM-entry exception error code field from the VMCS into
433	* the VMX transient structure.
434	*
435	* @returns VBox status code.
436	* @param pVmxTransient Pointer to the VMX transient structure.
437	*
438	* @remarks No-long-jump zone!!!
439	*/
440	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
441	{
442	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
443	AssertRCReturn(rc, rc);
444	return VINF_SUCCESS;
445	}
446
447
448	/**
449	* Reads the VM-exit interruption-information field from the VMCS into the VMX
450	* transient structure.
451	*
452	* @returns VBox status code.
453	* @param pVmxTransient Pointer to the VMX transient structure.
454	*/
455	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
456	{
457	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
458	{
459	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
460	AssertRCReturn(rc, rc);
461	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
462	}
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Reads the VM-exit interruption error code from the VMCS into the VMX
469	* transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVmxTransient Pointer to the VMX transient structure.
473	*/
474	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
475	{
476	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
477	{
478	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
479	AssertRCReturn(rc, rc);
480	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
481	}
482	return VINF_SUCCESS;
483	}
484
485
486	/**
487	* Reads the VM-exit instruction length field from the VMCS into the VMX
488	* transient structure.
489	*
490	* @returns VBox status code.
491	* @param pVmxTransient Pointer to the VMX transient structure.
492	*/
493	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
494	{
495	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
496	{
497	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
498	AssertRCReturn(rc, rc);
499	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
500	}
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the exit qualification from the VMCS into the VMX transient structure.
507	*
508	* @returns VBox status code.
509	* @param pVmxTransient Pointer to the VMX transient structure.
510	*/
511	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
512	{
513	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
514	{
515	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
516	AssertRCReturn(rc, rc);
517	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
518	}
519	return VINF_SUCCESS;
520	}
521
522
523	/**
524	* Reads the IDT-vectoring information field from the VMCS into the VMX
525	* transient structure.
526	*
527	* @returns VBox status code.
528	* @param pVmxTransient Pointer to the VMX transient structure.
529	*
530	* @remarks No-long-jump zone!!!
531	*/
532	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
533	{
534	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
535	{
536	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
537	AssertRCReturn(rc, rc);
538	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
539	}
540	return VINF_SUCCESS;
541	}
542
543
544	/**
545	* Reads the IDT-vectoring error code from the VMCS into the VMX
546	* transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVmxTransient Pointer to the VMX transient structure.
550	*/
551	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
552	{
553	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
554	{
555	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
556	AssertRCReturn(rc, rc);
557	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
558	}
559	return VINF_SUCCESS;
560	}
561
562
563	/**
564	* Enters VMX root mode operation on the current CPU.
565	*
566	* @returns VBox status code.
567	* @param pVM Pointer to the VM (optional, can be NULL, after
568	* a resume).
569	* @param HCPhysCpuPage Physical address of the VMXON region.
570	* @param pvCpuPage Pointer to the VMXON region.
571	*/
572	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
573	{
574	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
575	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
576
577	if (pVM)
578	{
579	/* Write the VMCS revision dword to the VMXON region. */
580	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
581	}
582
583	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
584	RTCCUINTREG fFlags = ASMIntDisableFlags();
585
586	/* Enable the VMX bit in CR4. */
587	RTCCUINTREG uCr4 = ASMGetCR4();
588	if (!(uCr4 & X86_CR4_VMXE))
589	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
590
591	/* Enter VMXON root mode. */
592	int rc = VMXEnable(HCPhysCpuPage);
593
594	/* Restore interrupts. */
595	ASMSetFlags(fFlags);
596	return rc;
597	}
598
599
600	/**
601	* Exits VMX root mode operation on the current CPU.
602	*
603	* @returns VBox status code.
604	*/
605	static int hmR0VmxLeaveRootMode(void)
606	{
607	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
608	RTCCUINTREG fFlags = ASMIntDisableFlags();
609	int rc = VINF_SUCCESS;
610
611	/* If we're for some reason not in VMX root mode, then don't leave it. */
612	if (ASMGetCR4() & X86_CR4_VMXE)
613	{
614	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
615	VMXDisable();
616	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
617	}
618	else
619	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
620
621	/* Restore interrupts. */
622	ASMSetFlags(fFlags);
623	return rc;
624	}
625
626
627	/**
628	* Allocates and maps one physically contiguous page. The allocated page is
629	* zero'd out. (Used by various VT-x structures).
630	*
631	* @returns IPRT status code.
632	* @param pMemObj Pointer to the ring-0 memory object.
633	* @param ppVirt Where to store the virtual address of the
634	* allocation.
635	* @param pPhys Where to store the physical address of the
636	* allocation.
637	*/
638	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
639	{
640	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
641	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
642	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
643
644	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
645	if (RT_FAILURE(rc))
646	return rc;
647	ppVirt = RTR0MemObjAddress(pMemObj);
648	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
649	ASMMemZero32(*ppVirt, PAGE_SIZE);
650	return VINF_SUCCESS;
651	}
652
653
654	/**
655	* Frees and unmaps an allocated physical page.
656	*
657	* @param pMemObj Pointer to the ring-0 memory object.
658	* @param ppVirt Where to re-initialize the virtual address of
659	* allocation as 0.
660	* @param pHCPhys Where to re-initialize the physical address of the
661	* allocation as 0.
662	*/
663	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
664	{
665	AssertPtr(pMemObj);
666	AssertPtr(ppVirt);
667	AssertPtr(pHCPhys);
668	if (*pMemObj != NIL_RTR0MEMOBJ)
669	{
670	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
671	AssertRC(rc);
672	*pMemObj = NIL_RTR0MEMOBJ;
673	*ppVirt = 0;
674	*pHCPhys = 0;
675	}
676	}
677
678
679	/**
680	* Worker function to free VT-x related structures.
681	*
682	* @returns IPRT status code.
683	* @param pVM Pointer to the VM.
684	*/
685	static void hmR0VmxStructsFree(PVM pVM)
686	{
687	for (VMCPUID i = 0; i < pVM->cCpus; i++)
688	{
689	PVMCPU pVCpu = &pVM->aCpus[i];
690	AssertPtr(pVCpu);
691
692	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
693	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
694
695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
696	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
697
698	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
699	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
700	}
701
702	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
703	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
704	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
705	#endif
706	}
707
708
709	/**
710	* Worker function to allocate VT-x related VM structures.
711	*
712	* @returns IPRT status code.
713	* @param pVM Pointer to the VM.
714	*/
715	static int hmR0VmxStructsAlloc(PVM pVM)
716	{
717	/*
718	* Initialize members up-front so we can cleanup properly on allocation failure.
719	*/
720	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
721	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
722	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
723	pVM->hm.s.vmx.HCPhys##a_Name = 0;
724
725	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
726	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
727	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
728	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
729
730	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
731	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
732	#endif
733	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
734
735	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
736	for (VMCPUID i = 0; i < pVM->cCpus; i++)
737	{
738	PVMCPU pVCpu = &pVM->aCpus[i];
739	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
740	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
741	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
742	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
743	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
744	}
745	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
746	#undef VMXLOCAL_INIT_VM_MEMOBJ
747
748	/*
749	* Allocate all the VT-x structures.
750	*/
751	int rc = VINF_SUCCESS;
752	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
753	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
754	if (RT_FAILURE(rc))
755	goto cleanup;
756	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
757	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
758	#endif
759
760	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
762	{
763	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
764	&pVM->hm.s.vmx.HCPhysApicAccess);
765	if (RT_FAILURE(rc))
766	goto cleanup;
767	}
768
769	/*
770	* Initialize per-VCPU VT-x structures.
771	*/
772	for (VMCPUID i =0; i < pVM->cCpus; i++)
773	{
774	PVMCPU pVCpu = &pVM->aCpus[i];
775	AssertPtr(pVCpu);
776
777	/* Allocate the VM control structure (VMCS). */
778	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
779	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
780	if (RT_FAILURE(rc))
781	goto cleanup;
782
783	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
784	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
785	{
786	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
787	&pVCpu->hm.s.vmx.HCPhysVirtApic);
788	if (RT_FAILURE(rc))
789	goto cleanup;
790	}
791
792	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
793	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
794	{
795	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
796	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
797	if (RT_FAILURE(rc))
798	goto cleanup;
799	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
800	}
801
802	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
803	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
804	if (RT_FAILURE(rc))
805	goto cleanup;
806
807	/* Allocate the VM-exit MSR-load page for the host MSRs. */
808	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
809	if (RT_FAILURE(rc))
810	goto cleanup;
811	}
812
813	return VINF_SUCCESS;
814
815	cleanup:
816	hmR0VmxStructsFree(pVM);
817	return rc;
818	}
819
820
821	/**
822	* Does global VT-x initialization (called during module initialization).
823	*
824	* @returns VBox status code.
825	*/
826	VMMR0DECL(int) VMXR0GlobalInit(void)
827	{
828	/* Setup the main VM exit handlers. */
829	/** @todo I'm told lookup table for function pointers is a bad idea in terms
830	* of performance. Make these static for now and not do change anything
831	* at runtime so we can easily switch to a switch-case approach later. */
832	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
833	#ifdef DEBUG
834	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
835	Assert(s_apfnVMExitHandlers[i]);
836	#endif
837	return VINF_SUCCESS;
838	}
839
840
841	/**
842	* Does global VT-x termination (called during module termination).
843	*/
844	VMMR0DECL(void) VMXR0GlobalTerm()
845	{
846	/* Nothing to do currently. */
847	}
848
849
850	/**
851	* Sets up and activates VT-x on the current CPU.
852	*
853	* @returns VBox status code.
854	* @param pCpu Pointer to the global CPU info struct.
855	* @param pVM Pointer to the VM (can be NULL after a host resume
856	* operation).
857	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
858	* fEnabledByHost is true).
859	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
860	* @a fEnabledByHost is true).
861	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
862	* enable VT-x/AMD-V on the host.
863	*/
864	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
865	{
866	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
867	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
868
869	if (!fEnabledByHost)
870	{
871	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
872	if (RT_FAILURE(rc))
873	return rc;
874	}
875
876	/*
877	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
878	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
879	* each time while reusing a VPID after hitting the MaxASID limit once.
880	*/
881	if ( pVM
882	&& pVM->hm.s.vmx.fVpid
883	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
884	{
885	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
886	pCpu->fFlushAsidBeforeUse = false;
887	}
888	else
889	pCpu->fFlushAsidBeforeUse = true;
890
891	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
892	++pCpu->cTlbFlushes;
893
894	return VINF_SUCCESS;
895	}
896
897
898	/**
899	* Deactivates VT-x on the current CPU.
900	*
901	* @returns VBox status code.
902	* @param pCpu Pointer to the global CPU info struct.
903	* @param pvCpuPage Pointer to the VMXON region.
904	* @param HCPhysCpuPage Physical address of the VMXON region.
905	*/
906	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
907	{
908	NOREF(pCpu);
909	NOREF(pvCpuPage);
910	NOREF(HCPhysCpuPage);
911
912	hmR0VmxLeaveRootMode();
913	return VINF_SUCCESS;
914	}
915
916
917	/**
918	* Sets the permission bits for the specified MSR in the MSR bitmap.
919	*
920	* @param pVCpu Pointer to the VMCPU.
921	* @param ulMSR The MSR value.
922	* @param enmRead Whether reading this MSR causes a VM-exit.
923	* @param enmWrite Whether writing this MSR causes a VM-exit.
924	*/
925	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
926	{
927	unsigned ulBit;
928	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
929
930	/*
931	* Layout:
932	* 0x000 - 0x3ff - Low MSR read bits
933	* 0x400 - 0x7ff - High MSR read bits
934	* 0x800 - 0xbff - Low MSR write bits
935	* 0xc00 - 0xfff - High MSR write bits
936	*/
937	if (ulMsr <= 0x00001FFF)
938	{
939	/* Pentium-compatible MSRs */
940	ulBit = ulMsr;
941	}
942	else if ( ulMsr >= 0xC0000000
943	&& ulMsr <= 0xC0001FFF)
944	{
945	/* AMD Sixth Generation x86 Processor MSRs */
946	ulBit = (ulMsr - 0xC0000000);
947	pbMsrBitmap += 0x400;
948	}
949	else
950	{
951	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
952	return;
953	}
954
955	Assert(ulBit <= 0x1fff);
956	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
957	ASMBitSet(pbMsrBitmap, ulBit);
958	else
959	ASMBitClear(pbMsrBitmap, ulBit);
960
961	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
962	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
963	else
964	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
965	}
966
967
968	/**
969	* Flushes the TLB using EPT.
970	*
971	* @returns VBox status code.
972	* @param pVM Pointer to the VM.
973	* @param pVCpu Pointer to the VMCPU.
974	* @param enmFlush Type of flush.
975	*/
976	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
977	{
978	AssertPtr(pVM);
979	Assert(pVM->hm.s.fNestedPaging);
980
981	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
982
983	uint64_t descriptor[2];
984	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
985	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
986
987	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
988	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
989	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
990	}
991
992
993	/**
994	* Flushes the TLB using VPID.
995	*
996	* @returns VBox status code.
997	* @param pVM Pointer to the VM.
998	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
999	* enmFlush).
1000	* @param enmFlush Type of flush.
1001	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1002	* on @a enmFlush).
1003	*/
1004	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1005	{
1006	AssertPtr(pVM);
1007	Assert(pVM->hm.s.vmx.fVpid);
1008
1009	uint64_t descriptor[2];
1010	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1011	{
1012	descriptor[0] = 0;
1013	descriptor[1] = 0;
1014	}
1015	else
1016	{
1017	AssertPtr(pVCpu);
1018	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1019	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1020	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1021	descriptor[1] = GCPtr;
1022	}
1023
1024	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1025	AssertMsg(rc == VINF_SUCCESS,
1026	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1027	if ( RT_SUCCESS(rc)
1028	&& pVCpu)
1029	{
1030	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1031	}
1032	}
1033
1034
1035	/**
1036	* Invalidates a guest page by guest virtual address. Only relevant for
1037	* EPT/VPID, otherwise there is nothing really to invalidate.
1038	*
1039	* @returns VBox status code.
1040	* @param pVM Pointer to the VM.
1041	* @param pVCpu Pointer to the VMCPU.
1042	* @param GCVirt Guest virtual address of the page to invalidate.
1043	*/
1044	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1045	{
1046	AssertPtr(pVM);
1047	AssertPtr(pVCpu);
1048	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1049
1050	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1051	if (!fFlushPending)
1052	{
1053	/*
1054	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1055	* See @bugref{6043} and @bugref{6177}.
1056	*
1057	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1058	* function maybe called in a loop with individual addresses.
1059	*/
1060	if (pVM->hm.s.vmx.fVpid)
1061	{
1062	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1063	{
1064	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1065	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1066	}
1067	else
1068	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1069	}
1070	else if (pVM->hm.s.fNestedPaging)
1071	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1072	}
1073
1074	return VINF_SUCCESS;
1075	}
1076
1077
1078	/**
1079	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1080	* otherwise there is nothing really to invalidate.
1081	*
1082	* @returns VBox status code.
1083	* @param pVM Pointer to the VM.
1084	* @param pVCpu Pointer to the VMCPU.
1085	* @param GCPhys Guest physical address of the page to invalidate.
1086	*/
1087	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1088	{
1089	LogFlowFunc(("%RGp\n", GCPhys));
1090
1091	/*
1092	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1093	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1094	* This function might be called in a loop.
1095	*/
1096	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1097	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1098	return VINF_SUCCESS;
1099	}
1100
1101
1102	/**
1103	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1104	* case where neither EPT nor VPID is supported by the CPU.
1105	*
1106	* @param pVM Pointer to the VM.
1107	* @param pVCpu Pointer to the VMCPU.
1108	*
1109	* @remarks Called with interrupts disabled.
1110	*/
1111	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1112	{
1113	NOREF(pVM);
1114	AssertPtr(pVCpu);
1115	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1116	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1117
1118	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1119	AssertPtr(pCpu);
1120
1121	pVCpu->hm.s.TlbShootdown.cPages = 0;
1122	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1123	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1124	pVCpu->hm.s.fForceTLBFlush = false;
1125	return;
1126	}
1127
1128
1129	/**
1130	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	* @remarks All references to "ASID" in this function pertains to "VPID" in
1135	* Intel's nomenclature. The reason is, to avoid confusion in compare
1136	* statements since the host-CPU copies are named "ASID".
1137	*
1138	* @remarks Called with interrupts disabled.
1139	* @todo Statistics.
1140	*/
1141	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1142	{
1143	AssertPtr(pVM);
1144	AssertPtr(pVCpu);
1145	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1146	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1147	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1148
1149	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1150	AssertPtr(pCpu);
1151
1152	/*
1153	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1154	* This can happen both for start & resume due to long jumps back to ring-3.
1155	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1156	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1157	*/
1158	bool fNewASID = false;
1159	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1160	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1161	{
1162	pVCpu->hm.s.fForceTLBFlush = true;
1163	fNewASID = true;
1164	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1165	}
1166
1167	/*
1168	* Check for explicit TLB shootdowns.
1169	*/
1170	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1171	{
1172	pVCpu->hm.s.fForceTLBFlush = true;
1173	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1174	}
1175
1176	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1177	if (pVCpu->hm.s.fForceTLBFlush)
1178	{
1179	if (fNewASID)
1180	{
1181	++pCpu->uCurrentAsid;
1182	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1183	{
1184	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1185	pCpu->cTlbFlushes++;
1186	pCpu->fFlushAsidBeforeUse = true;
1187	}
1188
1189	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1190	if (pCpu->fFlushAsidBeforeUse)
1191	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1192	}
1193	else
1194	{
1195	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1196	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1197	else
1198	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1199	}
1200
1201	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1202	pVCpu->hm.s.fForceTLBFlush = false;
1203	}
1204	else
1205	{
1206	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1207	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1208	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1209	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1210
1211	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1212	* not be executed. See hmQueueInvlPage() where it is commented
1213	* out. Support individual entry flushing someday. */
1214	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1215	{
1216	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1217
1218	/*
1219	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1220	* as supported by the CPU.
1221	*/
1222	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1223	{
1224	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1225	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1226	}
1227	else
1228	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1229	}
1230	else
1231	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1232	}
1233	pVCpu->hm.s.TlbShootdown.cPages = 0;
1234	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1235
1236	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1237	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1238	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1239	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1240	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1241	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1242
1243	/* Update VMCS with the VPID. */
1244	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1245	AssertRC(rc);
1246	}
1247
1248
1249	/**
1250	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1251	*
1252	* @returns VBox status code.
1253	* @param pVM Pointer to the VM.
1254	* @param pVCpu Pointer to the VMCPU.
1255	*
1256	* @remarks Called with interrupts disabled.
1257	* @todo Statistics.
1258	*/
1259	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1260	{
1261	AssertPtr(pVM);
1262	AssertPtr(pVCpu);
1263	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1264	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1265
1266	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1267	AssertPtr(pCpu);
1268
1269	/*
1270	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1271	* This can happen both for start & resume due to long jumps back to ring-3.
1272	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1273	*/
1274	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1275	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1276	{
1277	pVCpu->hm.s.fForceTLBFlush = true;
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1279	}
1280
1281	/* Check for explicit TLB shootdown flushes. */
1282	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1283	{
1284	pVCpu->hm.s.fForceTLBFlush = true;
1285	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1286	}
1287
1288	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1289	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1290
1291	if (pVCpu->hm.s.fForceTLBFlush)
1292	{
1293	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1294	pVCpu->hm.s.fForceTLBFlush = false;
1295	}
1296	else
1297	{
1298	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1299	* not be executed. See hmQueueInvlPage() where it is commented
1300	* out. Support individual entry flushing someday. */
1301	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1302	{
1303	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1305	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1306	}
1307	else
1308	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1309	}
1310
1311	pVCpu->hm.s.TlbShootdown.cPages= 0;
1312	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1313	}
1314
1315
1316	/**
1317	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1318	*
1319	* @returns VBox status code.
1320	* @param pVM Pointer to the VM.
1321	* @param pVCpu Pointer to the VMCPU.
1322	*
1323	* @remarks Called with interrupts disabled.
1324	* @todo Statistics.
1325	*/
1326	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1327	{
1328	AssertPtr(pVM);
1329	AssertPtr(pVCpu);
1330	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1331	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1332
1333	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1334
1335	/*
1336	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1337	* This can happen both for start & resume due to long jumps back to ring-3.
1338	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1339	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1340	*/
1341	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1342	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1343	{
1344	pVCpu->hm.s.fForceTLBFlush = true;
1345	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1346	}
1347
1348	/* Check for explicit TLB shootdown flushes. */
1349	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1350	{
1351	/*
1352	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1353	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1354	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1355	*/
1356	pVCpu->hm.s.fForceTLBFlush = true;
1357	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1358	}
1359
1360	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1361	if (pVCpu->hm.s.fForceTLBFlush)
1362	{
1363	++pCpu->uCurrentAsid;
1364	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1365	{
1366	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1367	pCpu->fFlushAsidBeforeUse = true;
1368	pCpu->cTlbFlushes++;
1369	}
1370
1371	pVCpu->hm.s.fForceTLBFlush = false;
1372	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1373	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1374	if (pCpu->fFlushAsidBeforeUse)
1375	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1376	}
1377	else
1378	{
1379	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1380	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1381	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1382	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1383
1384	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1385	* not be executed. See hmQueueInvlPage() where it is commented
1386	* out. Support individual entry flushing someday. */
1387	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1388	{
1389	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1390	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1391	{
1392	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1393	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1394	}
1395	else
1396	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1397	}
1398	else
1399	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1400	}
1401
1402	pVCpu->hm.s.TlbShootdown.cPages = 0;
1403	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1404
1405	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1406	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1407	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1408	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1409	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1410	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1411
1412	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1413	AssertRC(rc);
1414	}
1415
1416
1417	/**
1418	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1419	* TLB entries from the host TLB before VM-entry.
1420	*
1421	* @returns VBox status code.
1422	* @param pVM Pointer to the VM.
1423	*/
1424	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1425	{
1426	/*
1427	* Determine optimal flush type for nested paging.
1428	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1429	* guest execution (see hmR3InitFinalizeR0()).
1430	*/
1431	if (pVM->hm.s.fNestedPaging)
1432	{
1433	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1434	{
1435	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1436	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1437	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1438	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1439	else
1440	{
1441	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1442	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1443	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1444	}
1445
1446	/* Make sure the write-back cacheable memory type for EPT is supported. */
1447	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1448	{
1449	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1450	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1451	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1452	}
1453	}
1454	else
1455	{
1456	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1457	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1458	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1459	}
1460	}
1461
1462	/*
1463	* Determine optimal flush type for VPID.
1464	*/
1465	if (pVM->hm.s.vmx.fVpid)
1466	{
1467	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1468	{
1469	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1470	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1471	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1472	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1473	else
1474	{
1475	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1476	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1477	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1478	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1479	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1480	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1481	pVM->hm.s.vmx.fVpid = false;
1482	}
1483	}
1484	else
1485	{
1486	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1487	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1488	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1489	pVM->hm.s.vmx.fVpid = false;
1490	}
1491	}
1492
1493	/*
1494	* Setup the handler for flushing tagged-TLBs.
1495	*/
1496	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1497	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1498	else if (pVM->hm.s.fNestedPaging)
1499	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1500	else if (pVM->hm.s.vmx.fVpid)
1501	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1502	else
1503	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1504	return VINF_SUCCESS;
1505	}
1506
1507
1508	/**
1509	* Sets up pin-based VM-execution controls in the VMCS.
1510	*
1511	* @returns VBox status code.
1512	* @param pVM Pointer to the VM.
1513	* @param pVCpu Pointer to the VMCPU.
1514	*/
1515	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1516	{
1517	AssertPtr(pVM);
1518	AssertPtr(pVCpu);
1519
1520	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1521	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1522
1523	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1524	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1525	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1526
1527	/* Enable the VMX preemption timer. */
1528	if (pVM->hm.s.vmx.fUsePreemptTimer)
1529	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1530
1531	if ((val & zap) != val)
1532	{
1533	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1534	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1535	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1536	}
1537
1538	val &= zap;
1539	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1540	AssertRCReturn(rc, rc);
1541
1542	/* Update VCPU with the currently set pin-based VM-execution controls. */
1543	pVCpu->hm.s.vmx.u32PinCtls = val;
1544	return rc;
1545	}
1546
1547
1548	/**
1549	* Sets up processor-based VM-execution controls in the VMCS.
1550	*
1551	* @returns VBox status code.
1552	* @param pVM Pointer to the VM.
1553	* @param pVMCPU Pointer to the VMCPU.
1554	*/
1555	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1556	{
1557	AssertPtr(pVM);
1558	AssertPtr(pVCpu);
1559
1560	int rc = VERR_INTERNAL_ERROR_5;
1561	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1562	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1563
1564	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1565	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1566	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1567	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1568	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1569	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1570	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1571
1572	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1573	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1574	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1575	{
1576	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1577	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1578	}
1579
1580	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1581	if (!pVM->hm.s.fNestedPaging)
1582	{
1583	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1584	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1585	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1586	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1587	}
1588
1589	/* Use TPR shadowing if supported by the CPU. */
1590	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1591	{
1592	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1593	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1594	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1595	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1596
1597	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1598	/* CR8 writes causes a VM-exit based on TPR threshold. */
1599	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1600	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1601	}
1602	else
1603	{
1604	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1605	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1606	}
1607
1608	/* Use MSR-bitmaps if supported by the CPU. */
1609	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1610	{
1611	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1612
1613	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1614	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1615	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1616	AssertRCReturn(rc, rc);
1617
1618	/*
1619	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1620	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1621	*/
1622	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1623	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1624	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1625	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1626	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1627	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1628	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1629	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1630	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1631	}
1632
1633	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1634	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1635	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1636
1637	if ((val & zap) != val)
1638	{
1639	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1640	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1641	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1642	}
1643
1644	val &= zap;
1645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1646	AssertRCReturn(rc, rc);
1647
1648	/* Update VCPU with the currently set processor-based VM-execution controls. */
1649	pVCpu->hm.s.vmx.u32ProcCtls = val;
1650
1651	/*
1652	* Secondary processor-based VM-execution controls.
1653	*/
1654	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655	{
1656	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1657	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1658
1659	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1660
1661	if (pVM->hm.s.fNestedPaging)
1662	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1663	else
1664	{
1665	/*
1666	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1667	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1668	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1669	*/
1670	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1671	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1672	}
1673
1674	if (pVM->hm.s.vmx.fVpid)
1675	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1676
1677	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1678	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1679
1680	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1681	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1682	* done dynamically. */
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1684	{
1685	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1686	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1687	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1688	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1689	AssertRCReturn(rc, rc);
1690	}
1691
1692	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1693	{
1694	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1695	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1696	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1697	}
1698
1699	if ((val & zap) != val)
1700	{
1701	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1702	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1703	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1704	}
1705
1706	val &= zap;
1707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1708	AssertRCReturn(rc, rc);
1709
1710	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1711	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1712	}
1713
1714	return VINF_SUCCESS;
1715	}
1716
1717
1718	/**
1719	* Sets up miscellaneous (everything other than Pin & Processor-based
1720	* VM-execution) control fields in the VMCS.
1721	*
1722	* @returns VBox status code.
1723	* @param pVM Pointer to the VM.
1724	* @param pVCpu Pointer to the VMCPU.
1725	*/
1726	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1727	{
1728	AssertPtr(pVM);
1729	AssertPtr(pVCpu);
1730
1731	int rc = VERR_GENERAL_FAILURE;
1732
1733	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1735
1736	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1737
1738	/*
1739	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1740	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1741	* We thus use the exception bitmap to control it rather than use both.
1742	*/
1743	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1744	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1745
1746	/** @todo Explore possibility of using IO-bitmaps. */
1747	/* All IO & IOIO instructions cause VM-exits. */
1748	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1749	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1750
1751	/* Setup MSR autoloading/autostoring. */
1752	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1753	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1754	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1755	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1756	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1757	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1758
1759	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1760	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1761	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1762	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1763
1764	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1765	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1766
1767	/* Setup debug controls */
1768	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1769	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1770	AssertRCReturn(rc, rc);
1771	return rc;
1772	}
1773
1774
1775	/**
1776	* Sets up the initial exception bitmap in the VMCS based on static conditions
1777	* (i.e. conditions that cannot ever change at runtime).
1778	*
1779	* @returns VBox status code.
1780	* @param pVM Pointer to the VM.
1781	* @param pVCpu Pointer to the VMCPU.
1782	*/
1783	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1784	{
1785	AssertPtr(pVM);
1786	AssertPtr(pVCpu);
1787
1788	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1789
1790	uint32_t u32XcptBitmap = 0;
1791
1792	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1793	if (!pVM->hm.s.fNestedPaging)
1794	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1795
1796	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1797	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1798	AssertRCReturn(rc, rc);
1799	return rc;
1800	}
1801
1802
1803	/**
1804	* Sets up the initial guest-state mask. The guest-state mask is consulted
1805	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1806	* for the nested virtualization case (as it would cause a VM-exit).
1807	*
1808	* @param pVCpu Pointer to the VMCPU.
1809	*/
1810	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1811	{
1812	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1813	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1814	return VINF_SUCCESS;
1815	}
1816
1817
1818	/**
1819	* Does per-VM VT-x initialization.
1820	*
1821	* @returns VBox status code.
1822	* @param pVM Pointer to the VM.
1823	*/
1824	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1825	{
1826	LogFlowFunc(("pVM=%p\n", pVM));
1827
1828	int rc = hmR0VmxStructsAlloc(pVM);
1829	if (RT_FAILURE(rc))
1830	{
1831	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1832	return rc;
1833	}
1834
1835	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1836	{
1837	PVMCPU pVCpu = &pVM->aCpus[i];
1838
1839	/* Current guest paging mode. */
1840	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1841	}
1842
1843	return VINF_SUCCESS;
1844	}
1845
1846
1847	/**
1848	* Does per-VM VT-x termination.
1849	*
1850	* @returns VBox status code.
1851	* @param pVM Pointer to the VM.
1852	*/
1853	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1854	{
1855	LogFlowFunc(("pVM=%p\n", pVM));
1856
1857	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1858	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1859	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1860	#endif
1861	hmR0VmxStructsFree(pVM);
1862	return VINF_SUCCESS;
1863	}
1864
1865
1866	/**
1867	* Sets up the VM for execution under VT-x.
1868	* This function is only called once per-VM during initalization.
1869	*
1870	* @returns VBox status code.
1871	* @param pVM Pointer to the VM.
1872	*/
1873	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1874	{
1875	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1876	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1877
1878	LogFlowFunc(("pVM=%p\n", pVM));
1879
1880	/*
1881	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1882	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1883	*/
1884	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1885	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1886	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1887	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1888	{
1889	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1890	return VERR_INTERNAL_ERROR;
1891	}
1892
1893	/* Initialize these always, see hmR3InitFinalizeR0().*/
1894	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1895	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1896
1897	/* Setup the tagged-TLB flush handlers. */
1898	int rc = hmR0VmxSetupTaggedTlb(pVM);
1899	if (RT_FAILURE(rc))
1900	{
1901	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1902	return rc;
1903	}
1904
1905	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1906	{
1907	PVMCPU pVCpu = &pVM->aCpus[i];
1908	AssertPtr(pVCpu);
1909	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1910
1911	/* Set revision dword at the beginning of the VMCS structure. */
1912	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1913
1914	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1915	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1916	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1917	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1918
1919	/* Load this VMCS as the current VMCS. */
1920	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1921	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1922	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1923
1924	/* Setup the pin-based VM-execution controls. */
1925	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1926	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1927	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1928
1929	/* Setup the processor-based VM-execution controls. */
1930	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1931	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1932	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1933
1934	/* Setup the rest (miscellaneous) VM-execution controls. */
1935	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1936	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1937	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1938
1939	/* Setup the initial exception bitmap. */
1940	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1941	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1942	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1943
1944	/* Setup the initial guest-state mask. */
1945	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1946	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1947	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1948
1949	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1950	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1951	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1952	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1953	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1954	#endif
1955
1956	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1957	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	/* Update the last error record for this VCPU. */
1962	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1963	}
1964
1965	return VINF_SUCCESS;
1966	}
1967
1968
1969	/**
1970	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1971	* the VMCS.
1972	*
1973	* @returns VBox status code.
1974	* @param pVM Pointer to the VM.
1975	* @param pVCpu Pointer to the VMCPU.
1976	*/
1977	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1978	{
1979	RTCCUINTREG uReg = ASMGetCR0();
1980	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1981	AssertRCReturn(rc, rc);
1982
1983	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1984	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1985	if (VMX_IS_64BIT_HOST_MODE())
1986	{
1987	uint64_t uReg = hmR0Get64bitCR3();
1988	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
1989	}
1990	else
1991	#endif
1992	{
1993	uReg = ASMGetCR3();
1994	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
1995	}
1996	AssertRCReturn(rc, rc);
1997
1998	uReg = ASMGetCR4();
1999	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2000	AssertRCReturn(rc, rc);
2001	return rc;
2002	}
2003
2004
2005	/**
2006	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2007	* the host-state area in the VMCS.
2008	*
2009	* @returns VBox status code.
2010	* @param pVM Pointer to the VM.
2011	* @param pVCpu Pointer to the VMCPU.
2012	*/
2013	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2014	{
2015	int rc = VERR_INTERNAL_ERROR_5;
2016	RTSEL uSelCS = 0;
2017	RTSEL uSelSS = 0;
2018	RTSEL uSelDS = 0;
2019	RTSEL uSelES = 0;
2020	RTSEL uSelFS = 0;
2021	RTSEL uSelGS = 0;
2022	RTSEL uSelTR = 0;
2023
2024	/*
2025	* Host Selector registers.
2026	*/
2027	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2028	if (VMX_IS_64BIT_HOST_MODE())
2029	{
2030	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2031	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2032	}
2033	else
2034	{
2035	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2036	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2037	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2038	}
2039	#else
2040	uSelCS = ASMGetCS();
2041	uSelSS = ASMGetSS();
2042	#endif
2043
2044	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2045	/** @todo Verify if we have any platform that actually run with DS or ES with
2046	* RPL != 0 in kernel space. */
2047	uSelDS = 0;
2048	uSelES = 0;
2049	uSelFS = 0;
2050	uSelGS = 0;
2051	uSelTR = ASMGetTR();
2052
2053	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2054	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2055	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2056	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2057	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2058	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2059	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2060	Assert(uSelCS != 0);
2061	Assert(uSelTR != 0);
2062
2063	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2064	#if 0
2065	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2066	Assert(uSelSS != 0);
2067	#endif
2068
2069	/* Write these host selector fields into the host-state area in the VMCS. */
2070	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2071	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2072	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2073	#if 0
2074	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2075	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2076	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2077	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2078	#endif
2079	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2080	AssertRCReturn(rc, rc);
2081
2082	/*
2083	* Host GDTR and IDTR.
2084	*/
2085	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2086	* be safe to -not- save and restore GDTR and IDTR in the assembly
2087	* code and just do it here and don't care if the limits are zapped on
2088	* VM-exit. */
2089	RTGDTR Gdtr;
2090	RT_ZERO(Gdtr);
2091	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092	if (VMX_IS_64BIT_HOST_MODE())
2093	{
2094	X86XDTR64 Gtr64;
2095	X86XDTR64 Idtr64;
2096	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2097	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2098	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2099	Gdtr.cbGdt = Gdtr64.cb;
2100	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2101	}
2102	else
2103	#endif
2104	{
2105	RTIDTR Idtr;
2106	ASMGetGDTR(&Gdtr);
2107	ASMGetIDTR(&Idtr);
2108	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2109	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2110	}
2111	AssertRCReturn(rc, rc);
2112
2113	/*
2114	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2115	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2116	*/
2117	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2118	{
2119	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2120	return VERR_VMX_INVALID_HOST_STATE;
2121	}
2122
2123	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2124	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2125	if (VMX_IS_64BIT_HOST_MODE())
2126	{
2127	/* We need the 64-bit TR base for hybrid darwin. */
2128	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2129	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2130	}
2131	else
2132	#endif
2133	{
2134	uintptr_t uTRBase;
2135	#if HC_ARCH_BITS == 64
2136	uTRBase = X86DESC64_BASE(pDesc);
2137	#else
2138	uTRBase = X86DESC_BASE(pDesc);
2139	#endif
2140	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2141	}
2142	AssertRCReturn(rc, rc);
2143
2144	/*
2145	* Host FS base and GS base.
2146	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2147	* would take care of the bases. In 64-bit, the MSRs come into play.
2148	*/
2149	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2150	if (VMX_IS_64BIT_HOST_MODE())
2151	{
2152	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2153	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2154	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2155	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2156	AssertRCReturn(rc, rc);
2157	}
2158	#endif
2159	return rc;
2160	}
2161
2162
2163	/**
2164	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2165	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2166	* the host after every successful VM exit.
2167	*
2168	* @returns VBox status code.
2169	* @param pVM Pointer to the VM.
2170	* @param pVCpu Pointer to the VMCPU.
2171	*/
2172	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2173	{
2174	AssertPtr(pVCpu);
2175	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2176
2177	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2178	unsigned idxHostMsr = 0;
2179	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2180
2181	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2182	{
2183	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2184	pHostMsr->u32Reserved = 0;
2185	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2186	if (CPUMIsGuestInLongMode(pVCpu))
2187	{
2188	/* Must match the EFER value in our 64 bits switcher. */
2189	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2190	}
2191	else
2192	#endif
2193	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2194	pHostMsr++; idxHostMsr++;
2195	}
2196
2197	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2198	if (VMX_IS_64BIT_HOST_MODE())
2199	{
2200	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2201	pHostMsr->u32Reserved = 0;
2202	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2203	pHostMsr++; idxHostMsr++;
2204	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2205	pHostMsr->u32Reserved = 0;
2206	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2207	pHostMsr++; idxHostMsr++;
2208	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2209	pHostMsr->u32Reserved = 0;
2210	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2211	pHostMsr++; idxHostMsr++;
2212	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2213	#if 0
2214	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2215	pMsr->u32Reserved = 0;
2216	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2217	pHostMsr++; idxHostMsr++;
2218	#endif
2219	}
2220	#endif
2221
2222	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2223	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2224	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2225
2226	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2227	AssertRCReturn(rc, rc);
2228
2229	/*
2230	* Host Sysenter MSRs.
2231	*/
2232	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2233	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2234	if (VMX_IS_64BIT_HOST_MODE())
2235	{
2236	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2237	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2238	}
2239	else
2240	{
2241	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2242	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2243	}
2244	#elif HC_ARCH_BITS == 32
2245	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2246	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2247	#else
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2249	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2250	#endif
2251	AssertRCReturn(rc, rc);
2252
2253	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2254	* hmR0VmxSetupExitCtls() !! */
2255	return rc;
2256	}
2257
2258
2259	/**
2260	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2261	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2262	* controls".
2263	*
2264	* @returns VBox status code.
2265	* @param pVM Pointer to the VM.
2266	* @param pVCpu Pointer to the VMCPU.
2267	* @param pCtx Pointer to the guest-CPU context.
2268	*
2269	* @remarks No-long-jump zone!!!
2270	*/
2271	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2272	{
2273	int rc = VINF_SUCCESS;
2274	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2275	{
2276	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2277	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2278
2279	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2280	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2281
2282	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2283	if (CPUMIsGuestInLongModeEx(pCtx))
2284	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2285	else
2286	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2287
2288	/*
2289	* The following should not be set (since we're not in SMM mode):
2290	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2291	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2292	*/
2293
2294	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2295	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2296	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2297
2298	if ((val & zap) != val)
2299	{
2300	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2301	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2302	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2303	}
2304
2305	val &= zap;
2306	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2307	AssertRCReturn(rc, rc);
2308
2309	/* Update VCPU with the currently set VM-exit controls. */
2310	pVCpu->hm.s.vmx.u32EntryCtls = val;
2311	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2312	}
2313	return rc;
2314	}
2315
2316
2317	/**
2318	* Sets up the VM-exit controls in the VMCS.
2319	*
2320	* @returns VBox status code.
2321	* @param pVM Pointer to the VM.
2322	* @param pVCpu Pointer to the VMCPU.
2323	*/
2324	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2325	{
2326	int rc = VINF_SUCCESS;
2327	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2328	{
2329	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2330	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2331
2332	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2333	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2334
2335	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2336	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2337	if (VMX_IS_64BIT_HOST_MODE())
2338	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2339	else
2340	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2341	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2342	if (CPUMIsGuestInLongModeEx(pCtx))
2343	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2344	else
2345	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2346	#endif
2347
2348	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2349	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2350
2351	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2352	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2353	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2354	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2355	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2356
2357	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2358	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2359
2360	if ((val & zap) != val)
2361	{
2362	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2363	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2364	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2365	}
2366
2367	val &= zap;
2368	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2369
2370	/* Update VCPU with the currently set VM-exit controls. */
2371	pVCpu->hm.s.vmx.u32ExitCtls = val;
2372	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2373	}
2374	return rc;
2375	}
2376
2377
2378	/**
2379	* Loads the guest APIC and related state.
2380	*
2381	* @returns VBox status code.
2382	* @param pVM Pointer to the VM.
2383	* @param pVCpu Pointer to the VMCPU.
2384	* @param pCtx Pointer to the guest-CPU context.
2385	*/
2386	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2387	{
2388	int rc = VINF_SUCCESS;
2389	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2390	{
2391	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2392	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2393	{
2394	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395
2396	bool fPendingIntr = false;
2397	uint8_t u8GuestTpr = 0;
2398	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2399	AssertRCReturn(rc, rc);
2400
2401	/*
2402	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2403	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2404	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2405	* the interrupt when we VM-exit for other reasons.
2406	*/
2407	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2408	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2409	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2410	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2411
2412	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2413	AssertRCReturn(rc, rc);
2414
2415	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2416	if (pVM->hm.s.fTPRPatchingActive)
2417	{
2418	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2419	pCtx->msrLSTAR = u8GuestTpr;
2420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2421	{
2422	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2423	if (fPendingIntr)
2424	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2425	else
2426	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427	}
2428	}
2429	}
2430
2431	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2432	}
2433	return rc;
2434	}
2435
2436
2437	/**
2438	* Loads the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
2439	* into the guest-state area in the VMCS.
2440	*
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2444	* out-of-sync. Make sure to update the required fields
2445	* before using them.
2446	*
2447	* @remarks No-long-jump zone!!!
2448	*/
2449	DECLINLINE(void) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2450	{
2451	/*
2452	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2453	* inhibit interrupts or clear any existing interrupt-inhibition.
2454	*/
2455	uint32_t uIntrState = 0;
2456	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2457	{
2458	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2459	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS)),
2460	("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2461	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2462	{
2463	/*
2464	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2465	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2466	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
2467	*/
2468	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2469	}
2470	else if (pMixedCtx->eflags.Bits.u1IF)
2471	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2472	else
2473	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2474	}
2475
2476	Assert(!(uIntrState & 0xfffffff0)); /* Bits 31:4 MBZ. */
2477	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2478	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2479	AssertRC(rc);
2480	}
2481
2482
2483	/**
2484	* Loads the guest's RIP into the guest-state area in the VMCS.
2485	*
2486	* @returns VBox status code.
2487	* @param pVM Pointer to the VM.
2488	* @param pVCpu Pointer to the VMCPU.
2489	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2490	* out-of-sync. Make sure to update the required fields
2491	* before using them.
2492	*
2493	* @remarks No-long-jump zone!!!
2494	*/
2495	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2496	{
2497	int rc = VINF_SUCCESS;
2498	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2499	{
2500	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2501	AssertRCReturn(rc, rc);
2502	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2503	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2504	}
2505	return rc;
2506	}
2507
2508
2509	/**
2510	* Loads the guest's RSP into the guest-state area in the VMCS.
2511	*
2512	* @returns VBox status code.
2513	* @param pVM Pointer to the VM.
2514	* @param pVCpu Pointer to the VMCPU.
2515	* @param pCtx Pointer to the guest-CPU context.
2516	*
2517	* @remarks No-long-jump zone!!!
2518	*/
2519	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2520	{
2521	int rc = VINF_SUCCESS;
2522	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2523	{
2524	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2525	AssertRCReturn(rc, rc);
2526	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2527	}
2528	return rc;
2529	}
2530
2531
2532	/**
2533	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2534	*
2535	* @returns VBox status code.
2536	* @param pVM Pointer to the VM.
2537	* @param pVCpu Pointer to the VMCPU.
2538	* @param pCtx Pointer to the guest-CPU context.
2539	*
2540	* @remarks No-long-jump zone!!!
2541	*/
2542	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2543	{
2544	int rc = VINF_SUCCESS;
2545	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2546	{
2547	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2548	Let us assert it as such and use native-width VMWRITE. */
2549	X86RFLAGS uRFlags = pCtx->rflags;
2550	Assert(uRFlags.u64 >> 32 == 0);
2551	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2552	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2553
2554	/*
2555	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2556	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2557	*/
2558	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2559	{
2560	Assert(pVM->hm.s.vmx.pRealModeTSS);
2561	Assert(PDMVmmDevHeapIsEnabled(pVM));
2562	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2563	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2564	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2565	}
2566
2567	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2568	AssertRCReturn(rc, rc);
2569
2570	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2571	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2572	}
2573	return rc;
2574	}
2575
2576
2577	/**
2578	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2579	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2580	* assembly code.
2581	*
2582	* @returns VBox status code.
2583	* @param pVM Pointer to the VM.
2584	* @param pVCpu Pointer to the VMCPU.
2585	* @param pCtx Pointer to the guest-CPU context.
2586	*
2587	* @remarks No-long-jump zone!!!
2588	*/
2589	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2590	{
2591	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2592	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2593	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2594	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2595	AssertRCReturn(rc, rc);
2596	return rc;
2597	}
2598
2599
2600	/**
2601	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2602	* in the VMCS.
2603	*
2604	* @returns VBox status code.
2605	* @param pVM Pointer to the VM.
2606	* @param pVCpu Pointer to the VMCPU.
2607	* @param pCtx Pointer to the guest-CPU context.
2608	*
2609	* @remarks No-long-jump zone!!!
2610	*/
2611	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2612	{
2613	int rc = VINF_SUCCESS;
2614
2615	/*
2616	* Guest CR0.
2617	* Guest FPU.
2618	*/
2619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2620	{
2621	uint64_t u64GuestCR0 = pCtx->cr0;
2622
2623	/* The guest's view (read access) of its CR0 is unblemished. */
2624	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2625	AssertRCReturn(rc, rc);
2626	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2627
2628	/* Setup VT-x's view of the guest CR0. */
2629	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2630	if (pVM->hm.s.fNestedPaging)
2631	{
2632	if (CPUMIsGuestPagingEnabledEx(pCtx))
2633	{
2634	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2635	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2636	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2637	}
2638	else
2639	{
2640	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2641	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2643	}
2644
2645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2646	AssertRCReturn(rc, rc);
2647	}
2648	else
2649	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2650
2651	/*
2652	* Guest FPU bits.
2653	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2654	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2655	*/
2656	u64GuestCR0 \|= X86_CR0_NE;
2657	bool fInterceptNM = false;
2658	if (CPUMIsGuestFPUStateActive(pVCpu))
2659	{
2660	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2661	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2662	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2663	}
2664	else
2665	{
2666	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2667	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2668	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2669	}
2670
2671	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2672	bool fInterceptMF = false;
2673	if (!(pCtx->cr0 & X86_CR0_NE))
2674	fInterceptMF = true;
2675
2676	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2677	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2678	{
2679	Assert(PDMVmmDevHeapIsEnabled(pVM));
2680	Assert(pVM->hm.s.vmx.pRealModeTSS);
2681	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2682	fInterceptNM = true;
2683	fInterceptMF = true;
2684	}
2685	else
2686	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2687
2688	if (fInterceptNM)
2689	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2690	else
2691	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2692
2693	if (fInterceptMF)
2694	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2695	else
2696	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2697
2698	/* Additional intercepts for debugging, define these yourself explicitly. */
2699	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2700	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2701	\| RT_BIT(X86_XCPT_DB)
2702	\| RT_BIT(X86_XCPT_DE)
2703	\| RT_BIT(X86_XCPT_NM)
2704	\| RT_BIT(X86_XCPT_UD)
2705	\| RT_BIT(X86_XCPT_NP)
2706	\| RT_BIT(X86_XCPT_SS)
2707	\| RT_BIT(X86_XCPT_GP)
2708	\| RT_BIT(X86_XCPT_PF)
2709	\| RT_BIT(X86_XCPT_MF);
2710	#elif defined(VBOX_ALWAYS_TRAP_PF)
2711	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2712	#endif
2713
2714	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2715
2716	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2717	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2718	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2719	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2720	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2721	else
2722	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2723
2724	u64GuestCR0 \|= uSetCR0;
2725	u64GuestCR0 &= uZapCR0;
2726	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2727
2728	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2729	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2730	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2731	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2732
2733	/*
2734	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2735	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2736	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2737	*/
2738	uint64_t u64CR0Mask = 0;
2739	u64CR0Mask = X86_CR0_PE
2740	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2741	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2742	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2743	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2744	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2745
2746	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2747	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2748	u64CR0Mask &= ~X86_CR0_PE;
2749
2750	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2751	if (fInterceptNM)
2752	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2753	else
2754	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2755
2756	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2757	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2758	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2759	AssertRCReturn(rc, rc);
2760
2761	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2762	}
2763
2764	/*
2765	* Guest CR2.
2766	* It's always loaded in the assembler code. Nothing to do here.
2767	*/
2768
2769	/*
2770	* Guest CR3.
2771	*/
2772	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2773	{
2774	uint64_t u64GuestCR3 = 0;
2775	if (pVM->hm.s.fNestedPaging)
2776	{
2777	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2778
2779	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2780	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2781	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2782	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2783
2784	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2785	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2786	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2787
2788	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2789	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2790	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2791	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2792
2793	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2794	AssertRCReturn(rc, rc);
2795	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2796
2797	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2798	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2799	{
2800	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2801	if (CPUMIsGuestInPAEModeEx(pCtx))
2802	{
2803	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2804	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2805	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2806	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2807	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2808	AssertRCReturn(rc, rc);
2809	}
2810
2811	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2812	have Unrestricted Execution to handle the guest when it's not using paging. */
2813	u64GuestCR3 = pCtx->cr3;
2814	}
2815	else
2816	{
2817	/*
2818	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2819	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2820	* EPT takes care of translating it to host-physical addresses.
2821	*/
2822	RTGCPHYS GCPhys;
2823	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2824	Assert(PDMVmmDevHeapIsEnabled(pVM));
2825
2826	/* We obtain it here every time as the guest could have relocated this PCI region. */
2827	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2828	AssertRCReturn(rc, rc);
2829
2830	u64GuestCR3 = GCPhys;
2831	}
2832	}
2833	else
2834	{
2835	/* Non-nested paging case, just use the hypervisor's CR3. */
2836	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2837	}
2838
2839	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2840	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2841	AssertRCReturn(rc, rc);
2842
2843	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2844	}
2845
2846	/*
2847	* Guest CR4.
2848	*/
2849	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2850	{
2851	uint64_t u64GuestCR4 = pCtx->cr4;
2852
2853	/* The guest's view of its CR4 is unblemished. */
2854	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2855	AssertRCReturn(rc, rc);
2856	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2857
2858	/* Setup VT-x's view of the guest CR4. */
2859	/*
2860	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2861	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2862	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2863	*/
2864	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2865	{
2866	Assert(pVM->hm.s.vmx.pRealModeTSS);
2867	Assert(PDMVmmDevHeapIsEnabled(pVM));
2868	u64GuestCR4 &= ~X86_CR4_VME;
2869	}
2870
2871	if (pVM->hm.s.fNestedPaging)
2872	{
2873	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2874	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2875	{
2876	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2877	u64GuestCR4 \|= X86_CR4_PSE;
2878	/* Our identity mapping is a 32 bits page directory. */
2879	u64GuestCR4 &= ~X86_CR4_PAE;
2880	}
2881	/* else use guest CR4.*/
2882	}
2883	else
2884	{
2885	/*
2886	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2887	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2888	*/
2889	switch (pVCpu->hm.s.enmShadowMode)
2890	{
2891	case PGMMODE_REAL: /* Real-mode. */
2892	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2893	case PGMMODE_32_BIT: /* 32-bit paging. */
2894	{
2895	u64GuestCR4 &= ~X86_CR4_PAE;
2896	break;
2897	}
2898
2899	case PGMMODE_PAE: /* PAE paging. */
2900	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2901	{
2902	u64GuestCR4 \|= X86_CR4_PAE;
2903	break;
2904	}
2905
2906	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2907	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2908	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2909	break;
2910	#endif
2911	default:
2912	AssertFailed();
2913	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2914	}
2915	}
2916
2917	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2918	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2919	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2920	u64GuestCR4 \|= uSetCR4;
2921	u64GuestCR4 &= uZapCR4;
2922
2923	/* Write VT-x's view of the guest CR4 into the VMCS. */
2924	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2925	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2926
2927	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2928	uint64_t u64CR4Mask = 0;
2929	u64CR4Mask = X86_CR4_VME
2930	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2931	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2932	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2933	\| X86_CR4_VMXE;
2934	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2935	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2936	AssertRCReturn(rc, rc);
2937
2938	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2939	}
2940	return rc;
2941	}
2942
2943
2944	/**
2945	* Loads the guest debug registers into the guest-state area in the VMCS.
2946	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2947	*
2948	* @returns VBox status code.
2949	* @param pVM Pointer to the VM.
2950	* @param pVCpu Pointer to the VMCPU.
2951	* @param pCtx Pointer to the guest-CPU context.
2952	*
2953	* @remarks No-long-jump zone!!!
2954	*/
2955	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2956	{
2957	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2958	return VINF_SUCCESS;
2959
2960	#ifdef DEBUG
2961	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2962	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2963	{
2964	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2965	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2966	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2967	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2968	}
2969	#endif
2970
2971	int rc = VERR_INTERNAL_ERROR_5;
2972	bool fInterceptDB = false;
2973	bool fInterceptMovDRx = false;
2974	if (DBGFIsStepping(pVCpu))
2975	{
2976	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2977	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2978	{
2979	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2980	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2981	AssertRCReturn(rc, rc);
2982	Assert(fInterceptDB == false);
2983	}
2984	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2985	{
2986	Assert(fInterceptDB == false);
2987	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2988	* just continuing? */
2989	}
2990	else
2991	fInterceptDB = true;
2992	}
2993	else
2994	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2995
2996	/*
2997	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2998	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2999	* The same for the hypervisor DRx registers, priority is for the guest here.
3000	*/
3001	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3002	&& !CPUMIsGuestDebugStateActive(pVCpu))
3003	{
3004	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3005	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3006	AssertRCReturn(rc, rc);
3007	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3008	Assert(fInterceptMovDRx == false);
3009	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3010	}
3011	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3012	&& !CPUMIsHyperDebugStateActive(pVCpu))
3013	{
3014	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3015	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3016	AssertRCReturn(rc, rc);
3017	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3018	fInterceptMovDRx = true;
3019	}
3020	else
3021	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3022
3023	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3024	if (fInterceptDB)
3025	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3026	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3027	{
3028	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3029	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3030	#endif
3031	}
3032
3033	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3034	if (fInterceptMovDRx)
3035	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3036	else
3037	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3038
3039	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3040	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3041
3042	/* The guest's view of its DR7 is unblemished. */
3043	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3044
3045	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3046	return rc;
3047	}
3048
3049
3050	#ifdef DEBUG
3051	/**
3052	* Debug function to validate segment registers.
3053	*/
3054	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3055	{
3056	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3057	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3058	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3059	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3060	{
3061	/** @todo DPL checks for CS, SS. */
3062	/* Protected mode checks */
3063	/* CS */
3064	Assert(pCtx->cs.Attr.n.u1Present);
3065	Assert(!(pCtx->cs.Attr.u & 0xf00));
3066	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3067	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3068	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3069	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3070	\|\| (pCtx->cs.Attr.n.u1Granularity));
3071	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3072	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3073	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3074	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3075	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3076	else
3077	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3078	/* SS */
3079	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3080	{
3081	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3082	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3083	Assert(pCtx->ss.Attr.n.u1Present);
3084	Assert(!(pCtx->ss.Attr.u & 0xf00));
3085	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3086	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3087	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3088	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3089	\|\| (pCtx->ss.Attr.n.u1Granularity));
3090	}
3091	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3092	/* CR0 might not be up-to-date here always, hence disabled. */
3093	#if 0
3094	if (!pCtx->cr0 & X86_CR0_PE)
3095	Assert(!pCtx->ss.Attr.n.u2Dpl);
3096	#endif
3097	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3098	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3099	{
3100	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3101	Assert(pCtx->ds.Attr.n.u1Present);
3102	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3103	Assert(!(pCtx->ds.Attr.u & 0xf00));
3104	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3105	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3106	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3107	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3108	\|\| (pCtx->ds.Attr.n.u1Granularity));
3109	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3110	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3111	}
3112	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3113	{
3114	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3115	Assert(pCtx->es.Attr.n.u1Present);
3116	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3117	Assert(!(pCtx->es.Attr.u & 0xf00));
3118	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3119	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3120	\|\| !(pCtx->es.Attr.n.u1Granularity));
3121	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3122	\|\| (pCtx->es.Attr.n.u1Granularity));
3123	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3124	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3125	}
3126	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3127	{
3128	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3129	Assert(pCtx->fs.Attr.n.u1Present);
3130	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3131	Assert(!(pCtx->fs.Attr.u & 0xf00));
3132	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3133	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3134	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3135	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3136	\|\| (pCtx->fs.Attr.n.u1Granularity));
3137	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3138	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3139	}
3140	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3141	{
3142	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3143	Assert(pCtx->gs.Attr.n.u1Present);
3144	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3145	Assert(!(pCtx->gs.Attr.u & 0xf00));
3146	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3147	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3148	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3149	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3150	\|\| (pCtx->gs.Attr.n.u1Granularity));
3151	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3152	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3153	}
3154	/* 64-bit capable CPUs. */
3155	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3156	Assert(!(pCtx->cs.u64Base >> 32));
3157	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3158	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3159	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3160	# endif
3161	}
3162	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3163	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3164	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3165	{
3166	/* Real and v86 mode checks. */
3167	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3168	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3169	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3170	{
3171	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3172	}
3173	else
3174	{
3175	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3176	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3177	}
3178
3179	/* CS */
3180	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3181	Assert(pCtx->cs.u32Limit == 0xffff);
3182	Assert(u32CSAttr == 0xf3);
3183	/* SS */
3184	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3185	Assert(pCtx->ss.u32Limit == 0xffff);
3186	Assert(u32SSAttr == 0xf3);
3187	/* DS */
3188	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3189	Assert(pCtx->ds.u32Limit == 0xffff);
3190	Assert(u32DSAttr == 0xf3);
3191	/* ES */
3192	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3193	Assert(pCtx->es.u32Limit == 0xffff);
3194	Assert(u32ESAttr == 0xf3);
3195	/* FS */
3196	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3197	Assert(pCtx->fs.u32Limit == 0xffff);
3198	Assert(u32FSAttr == 0xf3);
3199	/* GS */
3200	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3201	Assert(pCtx->gs.u32Limit == 0xffff);
3202	Assert(u32GSAttr == 0xf3);
3203	/* 64-bit capable CPUs. */
3204	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3205	Assert(!(pCtx->cs.u64Base >> 32));
3206	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3207	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3208	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3209	# endif
3210	}
3211	}
3212	#endif /* DEBUG */
3213
3214
3215	/**
3216	* Writes a guest segment register into the guest-state area in the VMCS.
3217	*
3218	* @returns VBox status code.
3219	* @param pVM Pointer to the VM.
3220	* @param pVCpu Pointer to the VMCPU.
3221	* @param idxSel Index of the selector in the VMCS.
3222	* @param idxLimit Index of the segment limit in the VMCS.
3223	* @param idxBase Index of the segment base in the VMCS.
3224	* @param idxAccess Index of the access rights of the segment in the VMCS.
3225	* @param pSelReg Pointer to the segment selector.
3226	* @param pCtx Pointer to the guest-CPU context.
3227	*
3228	* @remarks No-long-jump zone!!!
3229	*/
3230	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3231	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3232	{
3233	int rc;
3234	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3235	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3236	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3237	AssertRCReturn(rc, rc);
3238
3239	uint32_t u32Access = pSelReg->Attr.u;
3240	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3241	{
3242	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3243	u32Access = 0xf3;
3244	Assert(pVM->hm.s.vmx.pRealModeTSS);
3245	Assert(PDMVmmDevHeapIsEnabled(pVM));
3246	}
3247	else
3248	{
3249	/*
3250	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3251	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3252	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3253	* loaded in protected-mode have their attribute as 0.
3254	*/
3255	if (!u32Access)
3256	u32Access = VMX_SEL_UNUSABLE;
3257	}
3258
3259	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3260	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3261	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3262
3263	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3264	AssertRCReturn(rc, rc);
3265	return rc;
3266	}
3267
3268
3269	/**
3270	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3271	* into the guest-state area in the VMCS.
3272	*
3273	* @returns VBox status code.
3274	* @param pVM Pointer to the VM.
3275	* @param pVCPU Pointer to the VMCPU.
3276	* @param pCtx Pointer to the guest-CPU context.
3277	*
3278	* @remarks No-long-jump zone!!!
3279	* @remarks Requires RFLAGS (for debug assertions).
3280	*/
3281	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3282	{
3283	int rc = VERR_INTERNAL_ERROR_5;
3284
3285	/*
3286	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3287	*/
3288	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3289	{
3290	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3291	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3292	{
3293	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3294	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3295	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3296	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3297	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3298	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3299	}
3300
3301	#ifdef VBOX_WITH_REM
3302	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3303	{
3304	Assert(pVM->hm.s.vmx.pRealModeTSS);
3305	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3306	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3307	{
3308	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3309	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3310	&& enmGuestMode >= PGMMODE_PROTECTED)
3311	{
3312	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3313	in real-mode (e.g. OpenBSD 4.0) */
3314	REMFlushTBs(pVM);
3315	Log2(("Switch to protected mode detected!\n"));
3316	}
3317	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3318	}
3319	}
3320	#endif
3321	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3322	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3323	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3324	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3325	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3326	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3327	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3328	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3329	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3330	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3331	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3332	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3333	AssertRCReturn(rc, rc);
3334
3335	#ifdef DEBUG
3336	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3337	#endif
3338	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3339	}
3340
3341	/*
3342	* Guest TR.
3343	*/
3344	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3345	{
3346	/*
3347	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3348	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3349	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3350	*/
3351	uint16_t u16Sel = 0;
3352	uint32_t u32Limit = 0;
3353	uint64_t u64Base = 0;
3354	uint32_t u32AccessRights = 0;
3355
3356	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3357	{
3358	u16Sel = pCtx->tr.Sel;
3359	u32Limit = pCtx->tr.u32Limit;
3360	u64Base = pCtx->tr.u64Base;
3361	u32AccessRights = pCtx->tr.Attr.u;
3362	}
3363	else
3364	{
3365	Assert(pVM->hm.s.vmx.pRealModeTSS);
3366	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3367
3368	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3369	RTGCPHYS GCPhys;
3370	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3371	AssertRCReturn(rc, rc);
3372
3373	X86DESCATTR DescAttr;
3374	DescAttr.u = 0;
3375	DescAttr.n.u1Present = 1;
3376	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3377
3378	u16Sel = 0;
3379	u32Limit = HM_VTX_TSS_SIZE;
3380	u64Base = GCPhys; /* in real-mode phys = virt. */
3381	u32AccessRights = DescAttr.u;
3382	}
3383
3384	/* Validate. */
3385	Assert(!(u16Sel & RT_BIT(2)));
3386	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3387	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3388	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3389	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3390	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3391	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3392	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3393	Assert( (u32Limit & 0xfff) == 0xfff
3394	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3395	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3396	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3397
3398	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3399	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3400	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3401	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3402	AssertRCReturn(rc, rc);
3403
3404	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3405	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3406	}
3407
3408	/*
3409	* Guest GDTR.
3410	*/
3411	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3412	{
3413	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3414	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3415	AssertRCReturn(rc, rc);
3416
3417	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3418	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3419	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3420	}
3421
3422	/*
3423	* Guest LDTR.
3424	*/
3425	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3426	{
3427	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3428	uint32_t u32Access = 0;
3429	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3430	u32Access = VMX_SEL_UNUSABLE;
3431	else
3432	u32Access = pCtx->ldtr.Attr.u;
3433
3434	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3435	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3436	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3437	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3438	AssertRCReturn(rc, rc);
3439
3440	/* Validate. */
3441	if (!(u32Access & VMX_SEL_UNUSABLE))
3442	{
3443	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3444	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3445	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3446	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3447	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3448	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3449	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3450	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3451	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3452	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3453	}
3454
3455	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3456	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3457	}
3458
3459	/*
3460	* Guest IDTR.
3461	*/
3462	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3463	{
3464	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3465	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3466	AssertRCReturn(rc, rc);
3467
3468	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3469	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3470	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3471	}
3472
3473	/*
3474	* Guest FS & GS base MSRs.
3475	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3476	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3477	* are only available in 64-bit mode.
3478	*/
3479	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3480	* should not be necessary to do it in assembly again. */
3481	if (CPUMIsGuestInLongModeEx(pCtx))
3482	{
3483	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3484	{
3485	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3486	AssertRCReturn(rc, rc);
3487	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3488	}
3489
3490	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3491	{
3492	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3493	AssertRCReturn(rc, rc);
3494	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3495	}
3496	}
3497	else
3498	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3499
3500	return VINF_SUCCESS;
3501	}
3502
3503
3504	/**
3505	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3506	* areas. These MSRs will automatically be loaded to the host CPU on every
3507	* successful VM entry and stored from the host CPU on every successful VM exit.
3508	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3509	*
3510	* @returns VBox status code.
3511	* @param pVM Pointer to the VM.
3512	* @param pVCpu Pointer to the VMCPU.
3513	* @param pCtx Pointer to the guest-CPU context.
3514	*
3515	* @remarks No-long-jump zone!!!
3516	*/
3517	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3518	{
3519	AssertPtr(pVCpu);
3520	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3521
3522	/*
3523	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3524	*/
3525	int rc = VINF_SUCCESS;
3526	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3527	{
3528	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3529	unsigned cGuestMsrs = 0;
3530
3531	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3532	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3533	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3534	if (fSupportsNX \|\| fSupportsLongMode)
3535	{
3536	/** @todo support save IA32_EFER, i.e.
3537	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3538	* guest EFER need not be part of the VM-entry MSR-load area. */
3539	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3540	pGuestMsr->u32Reserved = 0;
3541	pGuestMsr->u64Value = pCtx->msrEFER;
3542	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3543	if (!CPUMIsGuestInLongModeEx(pCtx))
3544	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3545	pGuestMsr++; cGuestMsrs++;
3546	if (fSupportsLongMode)
3547	{
3548	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3549	pGuestMsr->u32Reserved = 0;
3550	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3551	pGuestMsr++; cGuestMsrs++;
3552	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3553	pGuestMsr->u32Reserved = 0;
3554	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3555	pGuestMsr++; cGuestMsrs++;
3556	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3557	pGuestMsr->u32Reserved = 0;
3558	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3559	pGuestMsr++; cGuestMsrs++;
3560	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3561	#if 0
3562	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3563	pGuestMsr->u32Reserved = 0;
3564	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3565	pGuestMsr++; cGuestMsrs++;
3566	#endif
3567	}
3568	}
3569
3570	/*
3571	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3572	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3573	*/
3574	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3575	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3576	{
3577	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3578	pGuestMsr->u32Reserved = 0;
3579	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3580	AssertRCReturn(rc, rc);
3581	pGuestMsr++; cGuestMsrs++;
3582	}
3583
3584	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3585	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3586	{
3587	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3588	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3589	}
3590
3591	/* Update the VCPU's copy of the guest MSR count. */
3592	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3593	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3594	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3595	AssertRCReturn(rc, rc);
3596
3597	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3598	}
3599
3600	/*
3601	* Guest Sysenter MSRs.
3602	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3603	* VM exits on WRMSRs for these MSRs.
3604	*/
3605	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3606	{
3607	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3608	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3609	}
3610	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3611	{
3612	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3613	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3614	}
3615	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3616	{
3617	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3618	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3619	}
3620	AssertRCReturn(rc, rc);
3621
3622	return rc;
3623	}
3624
3625
3626	/**
3627	* Loads the guest activity state into the guest-state area in the VMCS.
3628	*
3629	* @returns VBox status code.
3630	* @param pVM Pointer to the VM.
3631	* @param pVCpu Pointer to the VMCPU.
3632	* @param pCtx Pointer to the guest-CPU context.
3633	*
3634	* @remarks No-long-jump zone!!!
3635	*/
3636	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3637	{
3638	/** @todo See if we can make use of other states, e.g.
3639	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3640	int rc = VINF_SUCCESS;
3641	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3642	{
3643	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3644	AssertRCReturn(rc, rc);
3645	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3646	}
3647	return rc;
3648	}
3649
3650
3651	/**
3652	* Sets up the appropriate function to run guest code.
3653	*
3654	* @returns VBox status code.
3655	* @param pVM Pointer to the VM.
3656	* @param pVCpu Pointer to the VMCPU.
3657	* @param pCtx Pointer to the guest-CPU context.
3658	*
3659	* @remarks No-long-jump zone!!!
3660	* @todo change this to return void.
3661	*/
3662	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3663	{
3664	if (CPUMIsGuestInLongModeEx(pCtx))
3665	{
3666	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3667	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3668	#endif
3669	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3670	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3671	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3672	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3673	#else
3674	/* 64-bit host or hybrid host. */
3675	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3676	#endif
3677	}
3678	else
3679	{
3680	/* Guest is not in long mode, use the 32-bit handler. */
3681	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3682	}
3683	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3684	return VINF_SUCCESS;
3685	}
3686
3687
3688	/**
3689	* Wrapper for running the guest code in VT-x.
3690	*
3691	* @returns VBox strict status code.
3692	* @param pVM Pointer to the VM.
3693	* @param pVCpu Pointer to the VMCPU.
3694	* @param pCtx Pointer to the guest-CPU context.
3695	*
3696	* @remarks No-long-jump zone!!!
3697	*/
3698	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3699	{
3700	/*
3701	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3702	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3703	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3704	*/
3705	#ifdef VBOX_WITH_KERNEL_USING_XMM
3706	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3707	#else
3708	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3709	#endif
3710	}
3711
3712
3713	/**
3714	* Report world-switch error and dump some useful debug info.
3715	*
3716	* @param pVM Pointer to the VM.
3717	* @param pVCpu Pointer to the VMCPU.
3718	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3719	* @param pCtx Pointer to the guest-CPU context.
3720	* @param pVmxTransient Pointer to the VMX transient structure (only
3721	* exitReason updated).
3722	*/
3723	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3724	{
3725	Assert(pVM);
3726	Assert(pVCpu);
3727	Assert(pCtx);
3728	Assert(pVmxTransient);
3729	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3730
3731	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3732	switch (rcVMRun)
3733	{
3734	case VERR_VMX_INVALID_VMXON_PTR:
3735	AssertFailed();
3736	break;
3737	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3738	case VERR_VMX_UNABLE_TO_START_VM:
3739	case VERR_VMX_UNABLE_TO_RESUME_VM:
3740	{
3741	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3742	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3743	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3744	AssertRC(rc);
3745
3746	#ifdef VBOX_STRICT
3747	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3748	pVmxTransient->uExitReason));
3749	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3750	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3751	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3752	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3753	else
3754	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3755
3756	/* VMX control bits. */
3757	uint32_t u32Val;
3758	uint64_t u64Val;
3759	RTHCUINTREG uHCReg;
3760	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3761	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3762	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3763	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3764	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3765	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3766	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3767	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3768	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3769	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3770	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3771	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3772	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3773	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3774	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3775	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3777	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3778	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3779	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3780	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3781	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3782	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3783	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3784	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3785	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3786	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3787	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3788	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3789	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3790	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3791	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3792	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3793	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3794	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3795	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3796	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3797	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3798	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3799	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3800	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3801	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3802
3803	/* Guest bits. */
3804	RTGCUINTREG uGCReg;
3805	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3806	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3807	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3808	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3809	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3810	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3811	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3813
3814	/* Host bits. */
3815	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3816	Log(("Host CR0 %#RHr\n", uHCReg));
3817	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3818	Log(("Host CR3 %#RHr\n", uHCReg));
3819	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3820	Log(("Host CR4 %#RHr\n", uHCReg));
3821
3822	RTGDTR HostGdtr;
3823	PCX86DESCHC pDesc;
3824	ASMGetGDTR(&HostGdtr);
3825	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3826	Log(("Host CS %#08x\n", u32Val));
3827	if (u32Val < HostGdtr.cbGdt)
3828	{
3829	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3830	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3831	}
3832
3833	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3834	Log(("Host DS %#08x\n", u32Val));
3835	if (u32Val < HostGdtr.cbGdt)
3836	{
3837	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3838	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3839	}
3840
3841	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3842	Log(("Host ES %#08x\n", u32Val));
3843	if (u32Val < HostGdtr.cbGdt)
3844	{
3845	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3846	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3847	}
3848
3849	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3850	Log(("Host FS %#08x\n", u32Val));
3851	if (u32Val < HostGdtr.cbGdt)
3852	{
3853	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3854	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3855	}
3856
3857	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3858	Log(("Host GS %#08x\n", u32Val));
3859	if (u32Val < HostGdtr.cbGdt)
3860	{
3861	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3862	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3863	}
3864
3865	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3866	Log(("Host SS %#08x\n", u32Val));
3867	if (u32Val < HostGdtr.cbGdt)
3868	{
3869	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3870	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3871	}
3872
3873	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3874	Log(("Host TR %#08x\n", u32Val));
3875	if (u32Val < HostGdtr.cbGdt)
3876	{
3877	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3878	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3879	}
3880
3881	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3882	Log(("Host TR Base %#RHv\n", uHCReg));
3883	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3884	Log(("Host GDTR Base %#RHv\n", uHCReg));
3885	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3886	Log(("Host IDTR Base %#RHv\n", uHCReg));
3887	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3888	Log(("Host SYSENTER CS %#08x\n", u32Val));
3889	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3890	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3891	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3892	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3893	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3894	Log(("Host RSP %#RHv\n", uHCReg));
3895	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3896	Log(("Host RIP %#RHv\n", uHCReg));
3897	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3898	if (VMX_IS_64BIT_HOST_MODE())
3899	{
3900	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3901	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3902	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3903	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3904	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3905	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3906	}
3907	# endif
3908	#endif /* VBOX_STRICT */
3909	break;
3910	}
3911
3912	default:
3913	/* Impossible */
3914	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3915	break;
3916	}
3917	NOREF(pVM);
3918	}
3919
3920
3921	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3922	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3923	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3924	#endif
3925
3926	/**
3927	* Executes the specified handler in 64-bit mode.
3928	*
3929	* @returns VBox status code.
3930	* @param pVM Pointer to the VM.
3931	* @param pVCpu Pointer to the VMCPU.
3932	* @param pCtx Pointer to the guest CPU context.
3933	* @param pfnHandler Pointer to the RC handler function.
3934	* @param cbParam Number of parameters.
3935	* @param paParam Array of 32-bit parameters.
3936	*/
3937	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3938	uint32_t *paParam)
3939	{
3940	int rc, rc2;
3941	PHMGLOBLCPUINFO pCpu;
3942	RTHCPHYS HCPhysCpuPage;
3943	RTHCUINTREG uOldEFlags;
3944
3945	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3946	Assert(pfnHandler);
3947	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3948	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3949
3950	#ifdef VBOX_STRICT
3951	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3952	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3953
3954	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3955	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3956	#endif
3957
3958	/* Disable interrupts. */
3959	uOldEFlags = ASMIntDisableFlags();
3960
3961	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3962	RTCPUID idHostCpu = RTMpCpuId();
3963	CPUMR0SetLApic(pVM, idHostCpu);
3964	#endif
3965
3966	pCpu = HMR0GetCurrentCpu();
3967	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
3968
3969	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3970	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
3971
3972	/* Leave VMX Root Mode. */
3973	VMXDisable();
3974
3975	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3976
3977	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3978	CPUMSetHyperEIP(pVCpu, pfnHandler);
3979	for (int i = (int)cbParam - 1; i >= 0; i--)
3980	CPUMPushHyper(pVCpu, paParam[i]);
3981
3982	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3983
3984	/* Call the switcher. */
3985	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
3986	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3987
3988	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
3989	/* Make sure the VMX instructions don't cause #UD faults. */
3990	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3991
3992	/* Re-enter VMX Root Mode */
3993	rc2 = VMXEnable(HCPhysCpuPage);
3994	if (RT_FAILURE(rc2))
3995	{
3996	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3997	ASMSetFlags(uOldEFlags);
3998	return VERR_VMX_VMXON_FAILED;
3999	}
4000
4001	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4002	AssertRC(rc2);
4003	Assert(!(ASMGetFlags() & X86_EFL_IF));
4004	ASMSetFlags(uOldEFlags);
4005	return rc;
4006	}
4007
4008
4009	/**
4010	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4011	* supporting 64-bit guests.
4012	*
4013	* @returns VBox status code.
4014	* @param fResume Whether to VMLAUNCH or VMRESUME.
4015	* @param pCtx Pointer to the guest-CPU context.
4016	* @param pCache Pointer to the VMCS cache.
4017	* @param pVM Pointer to the VM.
4018	* @param pVCpu Pointer to the VMCPU.
4019	*/
4020	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4021	{
4022	uint32_t aParam[6];
4023	PHMGLOBLCPUINFO pCpu = NULL;
4024	RTHCPHYS HCPhysCpuPage = 0;
4025	int rc = VERR_INTERNAL_ERROR_5;
4026
4027	pCpu = HMR0GetCurrentCpu();
4028	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4029
4030	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4031	pCache->uPos = 1;
4032	pCache->interPD = PGMGetInterPaeCR3(pVM);
4033	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4034	#endif
4035
4036	#ifdef DEBUG
4037	pCache->TestIn.HCPhysCpuPage= 0;
4038	pCache->TestIn.HCPhysVmcs = 0;
4039	pCache->TestIn.pCache = 0;
4040	pCache->TestOut.HCPhysVmcs = 0;
4041	pCache->TestOut.pCache = 0;
4042	pCache->TestOut.pCtx = 0;
4043	pCache->TestOut.eflags = 0;
4044	#endif
4045
4046	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4047	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4048	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4049	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4050	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4051	aParam[5] = 0;
4052
4053	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4054	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4055	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4056	#endif
4057	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4058
4059	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4060	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4061	Assert(pCtx->dr[4] == 10);
4062	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4063	#endif
4064
4065	#ifdef DEBUG
4066	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4067	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4068	pVCpu->hm.s.vmx.HCPhysVmcs));
4069	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4070	pCache->TestOut.HCPhysVmcs));
4071	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4072	pCache->TestOut.pCache));
4073	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4074	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4075	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4076	pCache->TestOut.pCtx));
4077	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4078	#endif
4079	return rc;
4080	}
4081
4082
4083	/**
4084	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4085	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4086	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4087	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4088	*
4089	* @returns VBox status code.
4090	* @param pVM Pointer to the VM.
4091	* @param pVCpu Pointer to the VMCPU.
4092	*/
4093	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4094	{
4095	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4096	{ \
4097	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4098	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4099	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4100	}
4101
4102	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4103	{ \
4104	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4105	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4106	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4107	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4108	}
4109
4110	AssertPtr(pVM);
4111	AssertPtr(pVCpu);
4112	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4113
4114	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4115	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4116	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4117	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4118	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4119	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4120	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4121	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4122	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4123
4124	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4125	#if 0
4126	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4127	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4128	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4129	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4130	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4131	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4132	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4133	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4134	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4135	#endif
4136
4137	/* 32-bit guest-state fields. */
4138	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4142	/* Unused 32-bit guest-state fields. */
4143	#if 0
4144	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4145	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4146	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4147	#endif
4148
4149	/* Natural width guest-state fields. */
4150	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4151	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4152	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4153	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4154	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4155	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4156	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4157	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4158	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4159	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4160	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4161	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4162	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4166	/* Unused natural width guest-state fields. */
4167	#if 0
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4169	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4170	#endif
4171	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4172	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4173
4174	if (pVM->hm.s.fNestedPaging)
4175	{
4176	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4177	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4178	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4179	}
4180	else
4181	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4182
4183	#undef VMXLOCAL_INIT_VMCS_SELREG
4184	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4185	return VINF_SUCCESS;
4186	}
4187
4188
4189	/**
4190	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4191	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4192	* darwin, running 64-bit guests).
4193	*
4194	* @returns VBox status code.
4195	* @param pVCpu Pointer to the VMCPU.
4196	* @param idxField The VMCS field encoding.
4197	* @param u64Val 16, 32 or 64 bits value.
4198	*/
4199	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4200	{
4201	int rc;
4202	switch (idxField)
4203	{
4204	/*
4205	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4206	*/
4207	/* 64-bit Control fields. */
4208	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4209	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4210	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4211	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4212	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4213	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4214	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4215	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4216	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4217	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4218	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4219	case VMX_VMCS64_CTRL_EPTP_FULL:
4220	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4221	/* 64-bit Read-only data fields. */
4222	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4223	/* 64-bit Guest-state fields. */
4224	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4225	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4226	case VMX_VMCS64_GUEST_PAT_FULL:
4227	case VMX_VMCS64_GUEST_EFER_FULL:
4228	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4229	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4230	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4231	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4232	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4233	/* 64-bit Host-state fields. */
4234	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4235	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4236	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4237	{
4238	rc = VMXWriteVmcs32(idxField, u64Val);
4239	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4240	break;
4241	}
4242
4243	/*
4244	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4245	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4246	*/
4247	/* Natural-width Guest-state fields. */
4248	case VMX_VMCS_GUEST_CR0:
4249	case VMX_VMCS_GUEST_CR3:
4250	case VMX_VMCS_GUEST_CR4:
4251	case VMX_VMCS_GUEST_ES_BASE:
4252	case VMX_VMCS_GUEST_CS_BASE:
4253	case VMX_VMCS_GUEST_SS_BASE:
4254	case VMX_VMCS_GUEST_DS_BASE:
4255	case VMX_VMCS_GUEST_FS_BASE:
4256	case VMX_VMCS_GUEST_GS_BASE:
4257	case VMX_VMCS_GUEST_LDTR_BASE:
4258	case VMX_VMCS_GUEST_TR_BASE:
4259	case VMX_VMCS_GUEST_GDTR_BASE:
4260	case VMX_VMCS_GUEST_IDTR_BASE:
4261	case VMX_VMCS_GUEST_DR7:
4262	case VMX_VMCS_GUEST_RSP:
4263	case VMX_VMCS_GUEST_RIP:
4264	case VMX_VMCS_GUEST_RFLAGS:
4265	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4266	case VMX_VMCS_GUEST_SYSENTER_ESP:
4267	case VMX_VMCS_GUEST_SYSENTER_EIP:
4268	{
4269	if ((u64Val >> 32ULL) == 0)
4270	{
4271	/* If this field is 64-bit, VT-x will zero out the top bits. */
4272	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4273	}
4274	else
4275	{
4276	/* Assert that only the 32->64 switcher case should ever come here. */
4277	Assert(pVM->hm.s.fAllow64BitGuests);
4278	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4279	}
4280	break;
4281	}
4282
4283	default:
4284	{
4285	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4286	rc = VERR_INVALID_PARAMETER;
4287	break;
4288	}
4289	}
4290	AssertRCReturn(rc, rc);
4291	return rc;
4292	}
4293
4294
4295	/**
4296	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4297	* hosts (except darwin) for 64-bit guests.
4298	*
4299	* @param pVCpu Pointer to the VMCPU.
4300	* @param idxField The VMCS field encoding.
4301	* @param u64Val 16, 32 or 64 bits value.
4302	*/
4303	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4304	{
4305	AssertPtr(pVCpu);
4306	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4307
4308	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4309	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4310
4311	/* Make sure there are no duplicates. */
4312	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4313	{
4314	if (pCache->Write.aField[i] == idxField)
4315	{
4316	pCache->Write.aFieldVal[i] = u64Val;
4317	return VINF_SUCCESS;
4318	}
4319	}
4320
4321	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4322	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4323	pCache->Write.cValidEntries++;
4324	return VINF_SUCCESS;
4325	}
4326
4327
4328	/**
4329	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4330	*
4331	* @param pCache Pointer to the VMCS cache.
4332	*/
4333	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4334	{
4335	AssertPtr(pCache);
4336	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4337	{
4338	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4339	AssertRC(rc, rc);
4340	}
4341	pCache->Write.cValidEntries = 0;
4342	}
4343
4344
4345	/**
4346	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4347	*
4348	* @param pCache Pointer to the VMCS cache.
4349	* @remarks No-long-jump zone!!!
4350	*/
4351	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4352	{
4353	AssertPtr(pCache);
4354	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4355	{
4356	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4357	AssertRC(rc);
4358	}
4359	}
4360	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4361
4362
4363	/**
4364	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4365	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4366	* timer.
4367	*
4368	* @returns VBox status code.
4369	* @param pVM Pointer to the VM.
4370	* @param pVCpu Pointer to the VMCPU.
4371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4372	* out-of-sync. Make sure to update the required fields
4373	* before using them.
4374	* @remarks No-long-jump zone!!!
4375	*/
4376	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4377	{
4378	int rc = VERR_INTERNAL_ERROR_5;
4379	bool fOffsettedTsc = false;
4380	if (pVM->hm.s.vmx.fUsePreemptTimer)
4381	{
4382	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4383
4384	/* Make sure the returned values have sane upper and lower boundaries. */
4385	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4386	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4387	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4388	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4389
4390	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4391	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4392	}
4393	else
4394	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4395
4396	if (fOffsettedTsc)
4397	{
4398	uint64_t u64CurTSC = ASMReadTSC();
4399	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4400	{
4401	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4402	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4403
4404	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4405	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4406	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4407	}
4408	else
4409	{
4410	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4411	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4412	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4413	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4414	}
4415	}
4416	else
4417	{
4418	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4419	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4420	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4421	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4422	}
4423	}
4424
4425
4426	/**
4427	* Determines if an exception is a benign exception. Benign exceptions
4428	* are ones which cannot cause double-faults.
4429	*
4430	* @returns true if the exception is benign, false otherwise.
4431	* @param uVector The exception vector.
4432	*/
4433	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4434	{
4435	switch (uVector)
4436	{
4437	case X86_XCPT_DB:
4438	case X86_XCPT_NMI:
4439	case X86_XCPT_BP:
4440	case X86_XCPT_OF:
4441	case X86_XCPT_BR:
4442	case X86_XCPT_UD:
4443	case X86_XCPT_NM:
4444	case X86_XCPT_CO_SEG_OVERRUN:
4445	case X86_XCPT_MF:
4446	case X86_XCPT_AC:
4447	case X86_XCPT_MC:
4448	case X86_XCPT_XF:
4449	return true;
4450	default:
4451	return false;
4452	}
4453	}
4454
4455
4456	/**
4457	* Determines if an exception is a contributory exception. Contributory
4458	* exceptions are ones which can cause double-faults.
4459	*
4460	* @returns true if the exception is contributory, false otherwise.
4461	* @param uVector The exception vector.
4462	*/
4463	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4464	{
4465	switch (uVector)
4466	{
4467	case X86_XCPT_GP:
4468	case X86_XCPT_SS:
4469	case X86_XCPT_NP:
4470	case X86_XCPT_TS:
4471	case X86_XCPT_DE:
4472	return true;
4473	default:
4474	return false;
4475	}
4476	return false;
4477	}
4478
4479
4480	/**
4481	* Determines if we are intercepting any contributory exceptions.
4482	*
4483	* @returns true if we are intercepting any contributory exception, false
4484	* otherwise.
4485	* @param pVCpu Pointer to the VMCPU.
4486	*/
4487	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4488	{
4489	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4490	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4491	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4492	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4493	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4494	{
4495	return true;
4496	}
4497	return false;
4498	}
4499
4500
4501	/**
4502	* Handle a condition that occurred while delivering an event through the guest
4503	* IDT.
4504	*
4505	* @returns VBox status code (informational error codes included).
4506	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4507	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4508	* continue execution of the guest which will delivery the #DF.
4509	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4510	*
4511	* @param pVM Pointer to the VM.
4512	* @param pVCpu Pointer to the VMCPU.
4513	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4514	* out-of-sync. Make sure to update the required fields
4515	* before using them.
4516	* @param pVmxTransient Pointer to the VMX transient structure.
4517	*
4518	* @remarks No-long-jump zone!!!
4519	*/
4520	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4521	{
4522	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4523	AssertRCReturn(rc, rc);
4524	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4525	{
4526	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4527	AssertRCReturn(rc, rc);
4528
4529	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4530	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4531	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4532
4533	typedef enum
4534	{
4535	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4536	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4537	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4538	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4539	} VMXREFLECTXCPT;
4540
4541	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4542	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4543	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4544	{
4545	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4546	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4547	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4548	&& uExitVector == X86_XCPT_PF))
4549	{
4550	enmReflect = VMXREFLECTXCPT_XCPT;
4551	}
4552	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4553	&& uIdtVector == X86_XCPT_PF
4554	&& uExitVector == X86_XCPT_PF)
4555	{
4556	pVmxTransient->fVectoringPF = true;
4557	}
4558	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4559	&& hmR0VmxIsContributoryXcpt(uExitVector))
4560	{
4561	enmReflect = VMXREFLECTXCPT_DF;
4562	}
4563	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4564	&& uIdtVector == X86_XCPT_PF
4565	&& hmR0VmxIsContributoryXcpt(uExitVector))
4566	{
4567	enmReflect = VMXREFLECTXCPT_DF;
4568	}
4569	else if (uIdtVector == X86_XCPT_DF)
4570	enmReflect = VMXREFLECTXCPT_TF;
4571	}
4572	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4573	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4574	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4575	{
4576	/*
4577	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4578	* (whatever they are) as they reoccur when restarting the instruction.
4579	*/
4580	enmReflect = VMXREFLECTXCPT_XCPT;
4581	}
4582
4583	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4584	switch (enmReflect)
4585	{
4586	case VMXREFLECTXCPT_XCPT:
4587	{
4588	Assert(!pVCpu->hm.s.Event.fPending);
4589	pVCpu->hm.s.Event.fPending = true;
4590	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4591	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4592	{
4593	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4594	AssertRCReturn(rc, rc);
4595	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4596	}
4597	else
4598	pVCpu->hm.s.Event.u32ErrCode = 0;
4599	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4600	break;
4601	}
4602
4603	case VMXREFLECTXCPT_DF:
4604	{
4605	uint32_t u32IntrInfo;
4606	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4607	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4608	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4609
4610	Assert(!pVCpu->hm.s.Event.fPending);
4611	pVCpu->hm.s.Event.fPending = true;
4612	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4613	pVCpu->hm.s.Event.u32ErrCode = 0;
4614	rc = VINF_VMX_DOUBLE_FAULT;
4615	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4616	break;
4617	}
4618
4619	case VMXREFLECTXCPT_TF:
4620	{
4621	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4622	rc = VINF_EM_RESET;
4623	break;
4624	}
4625
4626	default: /* shut up gcc. */
4627	break;
4628	}
4629	}
4630	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4631	return rc;
4632	}
4633
4634
4635	/**
4636	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4637	*
4638	* @returns VBox status code.
4639	* @param pVM Pointer to the VM.
4640	* @param pVCpu Pointer to the VMCPU.
4641	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4642	* out-of-sync. Make sure to update the required fields
4643	* before using them.
4644	*
4645	* @remarks No-long-jump zone!!!
4646	*/
4647	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4648	{
4649	int rc = VINF_SUCCESS;
4650	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4651	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4652	{
4653	RTGCUINTREG uVal = 0;
4654	RTCCUINTREG uShadow = 0;
4655	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4656	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4657	AssertRCReturn(rc, rc);
4658	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4659	CPUMSetGuestCR0(pVCpu, uVal);
4660	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4661	}
4662	return rc;
4663	}
4664
4665
4666	/**
4667	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4668	*
4669	* @returns VBox status code.
4670	* @param pVM Pointer to the VM.
4671	* @param pVCpu Pointer to the VMCPU.
4672	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4673	* out-of-sync. Make sure to update the required fields
4674	* before using them.
4675	*
4676	* @remarks No-long-jump zone!!!
4677	*/
4678	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4679	{
4680	int rc = VINF_SUCCESS;
4681	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4682	{
4683	RTGCUINTREG uVal = 0;
4684	RTCCUINTREG uShadow = 0;
4685	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4686	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4687	AssertRCReturn(rc, rc);
4688	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4689	CPUMSetGuestCR4(pVCpu, uVal);
4690	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4691	}
4692	return rc;
4693	}
4694
4695
4696	/**
4697	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4698	*
4699	* @returns VBox status code.
4700	* @param pVM Pointer to the VM.
4701	* @param pVCpu Pointer to the VMCPU.
4702	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4703	* out-of-sync. Make sure to update the required fields
4704	* before using them.
4705	*
4706	* @remarks No-long-jump zone!!!
4707	*/
4708	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4709	{
4710	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4711	return VINF_SUCCESS;
4712
4713	RTGCUINTREG uVal = 0;
4714	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4715	AssertRCReturn(rc, rc);
4716	pMixedCtx->rip = uVal;
4717	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4718	return rc;
4719	}
4720
4721
4722	/**
4723	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4724	*
4725	* @returns VBox status code.
4726	* @param pVM Pointer to the VM.
4727	* @param pVCpu Pointer to the VMCPU.
4728	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4729	* out-of-sync. Make sure to update the required fields
4730	* before using them.
4731	*
4732	* @remarks No-long-jump zone!!!
4733	*/
4734	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735	{
4736	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4737	return VINF_SUCCESS;
4738
4739	RTGCUINTREG uVal = 0;
4740	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4741	AssertRCReturn(rc, rc);
4742	pMixedCtx->rsp = uVal;
4743	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4744	return rc;
4745	}
4746
4747
4748	/**
4749	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4750	*
4751	* @returns VBox status code.
4752	* @param pVM Pointer to the VM.
4753	* @param pVCpu Pointer to the VMCPU.
4754	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4755	* out-of-sync. Make sure to update the required fields
4756	* before using them.
4757	*
4758	* @remarks No-long-jump zone!!!
4759	*/
4760	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4761	{
4762	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4763	return VINF_SUCCESS;
4764
4765	RTGCUINTREG uVal = 0;
4766	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4767	AssertRCReturn(rc, rc);
4768	pMixedCtx->rflags.u64 = uVal;
4769
4770	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4771	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4772	{
4773	Assert(pVM->hm.s.vmx.pRealModeTSS);
4774	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4775	pMixedCtx->eflags.Bits.u1VM = 0;
4776	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4777	}
4778
4779	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4780	return rc;
4781	}
4782
4783
4784	/**
4785	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4786	* guest-CPU context.
4787	*/
4788	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4789	{
4790	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4791	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4792	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4793	AssertRCReturn(rc, rc);
4794	return rc;
4795	}
4796
4797
4798	/**
4799	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4800	* from the guest-state area in the VMCS.
4801	*
4802	* @param pVM Pointer to the VM.
4803	* @param pVCpu Pointer to the VMCPU.
4804	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4805	* out-of-sync. Make sure to update the required fields
4806	* before using them.
4807	*
4808	* @remarks No-long-jump zone!!!
4809	*/
4810	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4811	{
4812	uint32_t uIntrState = 0;
4813	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4814	AssertRC(rc);
4815
4816	if (!uIntrState)
4817	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4818	else
4819	{
4820	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4821	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4822	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4823	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* for hmR0VmxLoadGuestIntrState(). */
4824	AssertRC(rc);
4825	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4826	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4827	}
4828	}
4829
4830
4831	/**
4832	* Saves the guest's activity state.
4833	*
4834	* @returns VBox status code.
4835	* @param pVM Pointer to the VM.
4836	* @param pVCpu Pointer to the VMCPU.
4837	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4838	* out-of-sync. Make sure to update the required fields
4839	* before using them.
4840	*
4841	* @remarks No-long-jump zone!!!
4842	*/
4843	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4844	{
4845	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4846	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4847	return VINF_SUCCESS;
4848	}
4849
4850
4851	/**
4852	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4853	* the current VMCS into the guest-CPU context.
4854	*
4855	* @returns VBox status code.
4856	* @param pVM Pointer to the VM.
4857	* @param pVCpu Pointer to the VMCPU.
4858	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4859	* out-of-sync. Make sure to update the required fields
4860	* before using them.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4865	{
4866	int rc = VINF_SUCCESS;
4867	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4868	{
4869	uint32_t u32Val = 0;
4870	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4871	pMixedCtx->SysEnter.cs = u32Val;
4872	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4873	}
4874
4875	RTGCUINTREG uGCVal = 0;
4876	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4877	{
4878	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4879	pMixedCtx->SysEnter.eip = uGCVal;
4880	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4881	}
4882	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4883	{
4884	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4885	pMixedCtx->SysEnter.esp = uGCVal;
4886	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4887	}
4888	return rc;
4889	}
4890
4891
4892	/**
4893	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4894	* context.
4895	*
4896	* @returns VBox status code.
4897	* @param pVM Pointer to the VM.
4898	* @param pVCpu Pointer to the VMCPU.
4899	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4900	* out-of-sync. Make sure to update the required fields
4901	* before using them.
4902	*
4903	* @remarks No-long-jump zone!!!
4904	*/
4905	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4906	{
4907	RTGCUINTREG uVal = 0;
4908	int rc = VINF_SUCCESS;
4909	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4910	{
4911	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4912	pMixedCtx->fs.u64Base = uVal;
4913	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4914	}
4915	return rc;
4916	}
4917
4918
4919	/**
4920	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4921	* context.
4922	*
4923	* @returns VBox status code.
4924	* @param pVM Pointer to the VM.
4925	* @param pVCpu Pointer to the VMCPU.
4926	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4927	* out-of-sync. Make sure to update the required fields
4928	* before using them.
4929	*
4930	* @remarks No-long-jump zone!!!
4931	*/
4932	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4933	{
4934	RTGCUINTREG uVal = 0;
4935	int rc = VINF_SUCCESS;
4936	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4937	{
4938	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4939	pMixedCtx->gs.u64Base = uVal;
4940	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4941	}
4942	return rc;
4943	}
4944
4945
4946	/**
4947	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4948	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4949	*
4950	* @returns VBox status code.
4951	* @param pVM Pointer to the VM.
4952	* @param pVCpu Pointer to the VMCPU.
4953	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4954	* out-of-sync. Make sure to update the required fields
4955	* before using them.
4956	*
4957	* @remarks No-long-jump zone!!!
4958	*/
4959	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4960	{
4961	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
4962	return VINF_SUCCESS;
4963
4964	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
4965	{
4966	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4967	pMsr += i;
4968	switch (pMsr->u32IndexMSR)
4969	{
4970	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
4971	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
4972	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
4973	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
4974	#if 0
4975	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
4976	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
4977	#endif
4978	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
4979	default:
4980	{
4981	AssertFailed();
4982	return VERR_HM_UNEXPECTED_LD_ST_MSR;
4983	}
4984	}
4985	}
4986	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
4987	return VINF_SUCCESS;
4988	}
4989
4990
4991	/**
4992	* Saves the guest control registers from the current VMCS into the guest-CPU
4993	* context.
4994	*
4995	* @returns VBox status code.
4996	* @param pVM Pointer to the VM.
4997	* @param pVCpu Pointer to the VMCPU.
4998	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4999	* out-of-sync. Make sure to update the required fields
5000	* before using them.
5001	*
5002	* @remarks No-long-jump zone!!!
5003	*/
5004	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5005	{
5006	RTGCUINTREG uVal = 0;
5007	RTGCUINTREG uShadow = 0;
5008	int rc = VINF_SUCCESS;
5009
5010	/* Guest CR0. Guest FPU. */
5011	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5012	AssertRCReturn(rc, rc);
5013
5014	/* Guest CR4. */
5015	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5016	AssertRCReturn(rc, rc);
5017
5018	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5019	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5020	{
5021	if ( pVM->hm.s.fNestedPaging
5022	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5023	{
5024	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5025	AssertRCReturn(rc, rc);
5026	if (pMixedCtx->cr3 != uVal)
5027	{
5028	CPUMSetGuestCR3(pVCpu, uVal);
5029	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5030	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5031	}
5032
5033	/* We require EFER to check PAE mode. */
5034	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5035	AssertRCReturn(rc, rc);
5036
5037	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5038	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5039	{
5040	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5041	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5042	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5043	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5044	AssertRCReturn(rc, rc);
5045	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5046	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5047	}
5048	}
5049	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5050	}
5051	return rc;
5052	}
5053
5054
5055	/**
5056	* Reads a guest segment register from the current VMCS into the guest-CPU
5057	* context.
5058	*
5059	* @returns VBox status code.
5060	* @param idxSel Index of the selector in the VMCS.
5061	* @param idxLimit Index of the segment limit in the VMCS.
5062	* @param idxBase Index of the segment base in the VMCS.
5063	* @param idxAccess Index of the access rights of the segment in the VMCS.
5064	* @param pSelReg Pointer to the segment selector.
5065	*
5066	* @remarks No-long-jump zone!!!
5067	*/
5068	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5069	PCPUMSELREG pSelReg)
5070	{
5071	uint32_t u32Val = 0;
5072	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5073	pSelReg->Sel = (uint16_t)u32Val;
5074	pSelReg->ValidSel = (uint16_t)u32Val;
5075	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5076
5077	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5078	pSelReg->u32Limit = u32Val;
5079
5080	RTGCUINTREG uGCVal = 0;
5081	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5082	pSelReg->u64Base = uGCVal;
5083
5084	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5085	pSelReg->Attr.u = u32Val;
5086
5087	/*
5088	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5089	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5090	*/
5091	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5092	{
5093	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5094	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5095	}
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Saves the guest segment registers from the current VMCS into the guest-CPU
5102	* context.
5103	*
5104	* @returns VBox status code.
5105	* @param pVM Pointer to the VM.
5106	* @param pVCpu Pointer to the VMCPU.
5107	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5108	* out-of-sync. Make sure to update the required fields
5109	* before using them.
5110	*
5111	* @remarks No-long-jump zone!!!
5112	*/
5113	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5114	{
5115	int rc = VINF_SUCCESS;
5116
5117	/* Guest segment registers. */
5118	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5119	{
5120	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5121	AssertRCReturn(rc, rc);
5122
5123	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5124	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5125	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5126	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5127	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5128	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5129	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5130	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5131	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5132	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5133	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5134	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5135	AssertRCReturn(rc, rc);
5136
5137	/* Restore segment attributes for real-on-v86 mode hack. */
5138	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5139	{
5140	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5141	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5142	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5143	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5144	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5145	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5146	}
5147
5148	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5149	}
5150
5151	/* Guest LDTR. */
5152	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5153	{
5154	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5155	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5156	AssertRCReturn(rc, rc);
5157	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5158	}
5159
5160	/* Guest GDTR. */
5161	RTGCUINTREG uGCVal = 0;
5162	uint32_t u32Val = 0;
5163	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5164	{
5165	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5166	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5167	pMixedCtx->gdtr.pGdt = uGCVal;
5168	pMixedCtx->gdtr.cbGdt = u32Val;
5169	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5170	}
5171
5172	/* Guest IDTR. */
5173	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5174	{
5175	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5176	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5177	pMixedCtx->idtr.pIdt = uGCVal;
5178	pMixedCtx->idtr.cbIdt = u32Val;
5179	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5180	}
5181
5182	/* Guest TR. */
5183	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5184	{
5185	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5186	AssertRCReturn(rc, rc);
5187
5188	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5189	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5190	{
5191	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5192	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5193	AssertRCReturn(rc, rc);
5194	}
5195	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5196	}
5197	return rc;
5198	}
5199
5200
5201	/**
5202	* Saves the guest debug registers from the current VMCS into the guest-CPU
5203	* context.
5204	*
5205	* @returns VBox status code.
5206	* @param pVM Pointer to the VM.
5207	* @param pVCpu Pointer to the VMCPU.
5208	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5209	* out-of-sync. Make sure to update the required fields
5210	* before using them.
5211	*
5212	* @remarks No-long-jump zone!!!
5213	*/
5214	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5215	{
5216	int rc = VINF_SUCCESS;
5217	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5218	{
5219	RTGCUINTREG uVal;
5220	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5221	pMixedCtx->dr[7] = uVal;
5222
5223	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5224	}
5225	return rc;
5226	}
5227
5228
5229	/**
5230	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5231	*
5232	* @returns VBox status code.
5233	* @param pVM Pointer to the VM.
5234	* @param pVCpu Pointer to the VMCPU.
5235	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5236	* out-of-sync. Make sure to update the required fields
5237	* before using them.
5238	*
5239	* @remarks No-long-jump zone!!!
5240	*/
5241	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5242	{
5243	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5244	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5245	return VINF_SUCCESS;
5246	}
5247
5248
5249	/**
5250	* Saves the entire guest state from the currently active VMCS into the
5251	* guest-CPU context. This essentially VMREADs all guest-data.
5252	*
5253	* @returns VBox status code.
5254	* @param pVM Pointer to the VM.
5255	* @param pVCpu Pointer to the VMCPU.
5256	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5257	* out-of-sync. Make sure to update the required fields
5258	* before using them.
5259	*/
5260	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5261	{
5262	Assert(pVM);
5263	Assert(pVCpu);
5264	Assert(pMixedCtx);
5265
5266	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5267	return VINF_SUCCESS;
5268
5269	VMMRZCallRing3Disable(pVCpu);
5270
5271	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5272	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5273
5274	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5275	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5276
5277	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5278	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5279
5280	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5281	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5282
5283	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5284	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5285
5286	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5287	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5288
5289	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5290	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5291
5292	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5293	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5294
5295	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5296	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5297
5298	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5299	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5300
5301	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5302	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5303
5304	VMMRZCallRing3Enable(pVCpu);
5305	return rc;
5306	}
5307
5308
5309	/**
5310	* Check per-VM and per-VCPU force flag actions that require us to go back to
5311	* ring-3 for one reason or another.
5312	*
5313	* @returns VBox status code (information status code included).
5314	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5315	* ring-3.
5316	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5317	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5318	* interrupts)
5319	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5320	* all EMTs to be in ring-3.
5321	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5322	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5323	* to the EM loop.
5324	*
5325	* @param pVM Pointer to the VM.
5326	* @param pVCpu Pointer to the VMCPU.
5327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5328	* out-of-sync. Make sure to update the required fields
5329	* before using them.
5330	*/
5331	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5332	{
5333	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5334
5335	int rc = VERR_INTERNAL_ERROR_5;
5336	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5337	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5338	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5339	{
5340	/* We need the control registers now, make sure the guest-CPU context is updated. */
5341	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5342	AssertRCReturn(rc, rc);
5343
5344	/* Pending HM CR3 sync. */
5345	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5346	{
5347	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5348	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5349	}
5350	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5351	{
5352	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5353	AssertRCReturn(rc, rc);
5354	}
5355
5356	/* Pending PGM C3 sync. */
5357	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5358	{
5359	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5360	if (rc != VINF_SUCCESS)
5361	{
5362	AssertRC(rc);
5363	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5364	return rc;
5365	}
5366	}
5367
5368	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5369	/* -XXX- what was that about single stepping? */
5370	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5371	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5372	{
5373	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5374	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5375	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5376	return rc;
5377	}
5378
5379	/* Pending VM request packets, such as hardware interrupts. */
5380	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5381	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5382	{
5383	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5384	return VINF_EM_PENDING_REQUEST;
5385	}
5386
5387	/* Pending PGM pool flushes. */
5388	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5389	{
5390	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5391	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5392	}
5393
5394	/* Pending DMA requests. */
5395	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5396	{
5397	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5398	return VINF_EM_RAW_TO_R3;
5399	}
5400	}
5401
5402	/* Paranoia. */
5403	Assert(rc != VERR_EM_INTERPRETER);
5404	return VINF_SUCCESS;
5405	}
5406
5407
5408	/**
5409	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5410	* VT-x to execute any instruction.
5411	*
5412	* @param pvCpu Pointer to the VMCPU.
5413	*/
5414	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5415	{
5416	if (pVCpu->hm.s.Event.fPending)
5417	{
5418	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5419	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5420	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5421	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5422
5423	/* If a trap was already pending, we did something wrong! */
5424	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5425
5426	/* A page-fault exception during a page-fault would become a double-fault. */
5427	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5428	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5429
5430	TRPMEVENT enmTrapType;
5431	switch (uVectorType)
5432	{
5433	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5434	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5435	enmTrapType = TRPM_HARDWARE_INT;
5436	break;
5437	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5438	enmTrapType = TRPM_SOFTWARE_INT;
5439	break;
5440	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5441	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5442	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5443	enmTrapType = TRPM_TRAP;
5444	break;
5445	default:
5446	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5447	enmTrapType = TRPM_32BIT_HACK;
5448	break;
5449	}
5450	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5451	AssertRC(rc);
5452	if (fErrorCodeValid)
5453	TRPMSetErrorCode(pVCpu, uErrorCode);
5454
5455	/* Clear the VT-x state bits now that TRPM has the information. */
5456	pVCpu->hm.s.Event.fPending = false;
5457	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5458	AssertRC(rc);
5459	}
5460	}
5461
5462
5463	/**
5464	* Does the necessary state syncing before doing a longjmp to ring-3.
5465	*
5466	* @param pVM Pointer to the VM.
5467	* @param pVCpu Pointer to the VMCPU.
5468	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5469	* out-of-sync. Make sure to update the required fields
5470	* before using them.
5471	* @param rcExit The reason for exiting to ring-3. Can be
5472	* VINF_VMM_UNKNOWN_RING3_CALL.
5473	*
5474	* @remarks No-long-jmp zone!!!
5475	*/
5476	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5477	{
5478	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5479
5480	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5481	AssertRC(rc);
5482
5483	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5484	if (CPUMIsGuestFPUStateActive(pVCpu))
5485	{
5486	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5487	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5488	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5489	}
5490
5491	/* Restore debug registers if necessary and resync on next R0 reentry. */
5492	if (CPUMIsGuestDebugStateActive(pVCpu))
5493	{
5494	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5495	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5496	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5497	}
5498	else if (CPUMIsHyperDebugStateActive(pVCpu))
5499	{
5500	CPUMR0LoadHostDebugState(pVM, pVCpu);
5501	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5502	}
5503
5504	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5505	}
5506
5507
5508	/**
5509	* An action requires us to go back to ring-3. This function does the necessary
5510	* steps before we can safely return to ring-3. This is not the same as longjmps
5511	* to ring-3, this is voluntary.
5512	*
5513	* @param pVM Pointer to the VM.
5514	* @param pVCpu Pointer to the VMCPU.
5515	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5516	* out-of-sync. Make sure to update the required fields
5517	* before using them.
5518	* @param rcExit The reason for exiting to ring-3. Can be
5519	* VINF_VMM_UNKNOWN_RING3_CALL.
5520	*/
5521	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5522	{
5523	Assert(pVM);
5524	Assert(pVCpu);
5525	Assert(pMixedCtx);
5526	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5527
5528	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5529	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5530	return;
5531
5532	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5533	VMMRZCallRing3Disable(pVCpu);
5534	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5535
5536	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5537	hmR0VmxUpdateTRPMTrap(pVCpu);
5538
5539	/* Sync. the guest state. */
5540	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5541	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5542
5543	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5544	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5545	\| CPUM_CHANGED_LDTR
5546	\| CPUM_CHANGED_GDTR
5547	\| CPUM_CHANGED_IDTR
5548	\| CPUM_CHANGED_TR
5549	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5550
5551	/* On our way back from ring-3 the following needs to be done. */
5552	/** @todo This can change with preemption hooks. */
5553	if (rcExit == VINF_EM_RAW_INTERRUPT)
5554	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5555	else
5556	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5557
5558	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5559	VMMRZCallRing3Enable(pVCpu);
5560	}
5561
5562
5563	/**
5564	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5565	* longjump to ring-3 and possibly get preempted.
5566	*
5567	* @param pVCpu Pointer to the VMCPU.
5568	* @param enmOperation The operation causing the ring-3 longjump.
5569	* @param pvUser The user argument (pointer to the possibly
5570	* out-of-date guest-CPU context).
5571	*
5572	* @remarks Must never be called with @a enmOperation ==
5573	* VMMCALLRING3_VM_R0_ASSERTION.
5574	*/
5575	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5576	{
5577	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5578	Assert(pVCpu);
5579	Assert(pvUser);
5580	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5581	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5582
5583	VMMRZCallRing3Disable(pVCpu);
5584	Log(("hmR0VmxLongJmpToRing3\n"));
5585	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5586	VMMRZCallRing3Enable(pVCpu);
5587	}
5588
5589
5590	/**
5591	* Injects any pending TRPM trap into the VM by updating the VMCS.
5592	*
5593	* @returns VBox status code (informational status code included).
5594	* @param pVM Pointer to the VM.
5595	* @param pVCpu Pointer to the VMCPU.
5596	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5597	* out-of-sync. Make sure to update the required fields
5598	* before using them.
5599	*/
5600	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5601	{
5602	if (!TRPMHasTrap(pVCpu))
5603	return VINF_SUCCESS;
5604
5605	uint8_t u8Vector = 0;
5606	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5607	RTGCUINT uErrCode = 0;
5608
5609	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5610	AssertRCReturn(rc, rc);
5611	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5612
5613	rc = TRPMResetTrap(pVCpu);
5614	AssertRCReturn(rc, rc);
5615
5616	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5617	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5618	if (enmTrpmEvent == TRPM_TRAP)
5619	{
5620	switch (u8Vector)
5621	{
5622	case X86_XCPT_BP:
5623	case X86_XCPT_OF:
5624	{
5625	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5626	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5627	break;
5628	}
5629
5630	case X86_XCPT_DF:
5631	case X86_XCPT_TS:
5632	case X86_XCPT_NP:
5633	case X86_XCPT_SS:
5634	case X86_XCPT_GP:
5635	case X86_XCPT_PF:
5636	case X86_XCPT_AC:
5637	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5638	them which VT-x/VMM pushes to the guest stack. */
5639	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5640	/* no break! */
5641	default:
5642	{
5643	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5644	break;
5645	}
5646	}
5647	}
5648	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5649	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5650	else
5651	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5652
5653	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5654	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5655	}
5656
5657
5658	/**
5659	* Checks if there are any pending guest interrupts to be delivered and injects
5660	* them into the VM by updating the VMCS.
5661	*
5662	* @returns VBox status code (informational status codes included).
5663	* @param pVM Pointer to the VM.
5664	* @param pVCpu Pointer to the VMCPU.
5665	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5666	* out-of-sync. Make sure to update the required fields
5667	* before using them.
5668	*
5669	* @remarks Must be called after hmR0VmxLoadGuestIntrState().
5670	*/
5671	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5672	{
5673	/* First inject any pending HM interrupts. */
5674	if (pVCpu->hm.s.Event.fPending)
5675	{
5676	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5677	pVCpu->hm.s.Event.u32ErrCode);
5678	AssertRCReturn(rc, rc);
5679	pVCpu->hm.s.Event.fPending = false;
5680	return rc;
5681	}
5682
5683	/** @todo SMI. SMIs take priority over NMIs. */
5684
5685	/* NMI. NMIs take priority over regular interrupts . */
5686	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5687	{
5688	/* Construct an NMI interrupt and inject it into the VMCS. */
5689	RTGCUINTPTR uIntrInfo;
5690	uIntrInfo = X86_XCPT_NMI;
5691	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5692	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5693	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5694	AssertRCReturn(rc, rc);
5695	return rc;
5696	}
5697
5698	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5699	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5700	AssertRCReturn(rc, rc);
5701
5702	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5703	if (!TRPMHasTrap(pVCpu))
5704	{
5705	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5706	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5707	{
5708	/*
5709	* If the guest can receive interrupts now (interrupts enabled and no interrupt inhibition is active) convert
5710	* the PDM interrupt into a TRPM event and inject it.
5711	*/
5712	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5713	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5714	{
5715	uint8_t u8Interrupt = 0;
5716	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5717	if (RT_SUCCESS(rc))
5718	{
5719	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5720	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5721	AssertRCReturn(rc, rc);
5722	}
5723	else
5724	{
5725	/** @todo Does this actually happen? If not turn it into an assertion. */
5726	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5727	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5728	}
5729	}
5730	else if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5731	{
5732	/* Instruct VT-x to cause an interrupt-window exit as soon as the guest is ready to receive interrupts again. */
5733	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5735	AssertRCReturn(rc, rc);
5736	}
5737	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5738	}
5739	}
5740
5741	/* If interrupts can be delivered, inject it into the VM. */
5742	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5743	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5744	&& TRPMHasTrap(pVCpu))
5745	{
5746	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5747	AssertRCReturn(rc, rc);
5748	}
5749	return rc;
5750	}
5751
5752	/**
5753	* Injects an invalid-opcode (#UD) exception into the VM.
5754	*
5755	* @returns VBox status code (informational status code included).
5756	* @param pVM Pointer to the VM.
5757	* @param pVCpu Pointer to the VMCPU.
5758	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5759	* out-of-sync. Make sure to update the required fields
5760	* before using them.
5761	*/
5762	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5763	{
5764	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5765	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5766	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5767	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5768	}
5769
5770
5771	/**
5772	* Injects a double-fault (#DF) exception into the VM.
5773	*
5774	* @returns VBox status code (informational status code included).
5775	* @param pVM Pointer to the VM.
5776	* @param pVCpu Pointer to the VMCPU.
5777	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778	* out-of-sync. Make sure to update the required fields
5779	* before using them.
5780	*/
5781	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5782	{
5783	/* Inject the double-fault. */
5784	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5785	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5786	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5787	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5788	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5789	}
5790
5791
5792	/**
5793	* Injects a debug (#DB) exception into the VM.
5794	*
5795	* @returns VBox status code (informational status code included).
5796	* @param pVM Pointer to the VM.
5797	* @param pVCpu Pointer to the VMCPU.
5798	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5799	* out-of-sync. Make sure to update the required fields
5800	* before using them.
5801	*/
5802	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5803	{
5804	/* Inject the debug-exception. */
5805	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5806	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5807	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5808	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5809	}
5810
5811
5812	/**
5813	* Injects a overflow (#OF) exception into the VM.
5814	*
5815	* @returns VBox status code (informational status code included).
5816	* @param pVM Pointer to the VM.
5817	* @param pVCpu Pointer to the VMCPU.
5818	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5819	* out-of-sync. Make sure to update the required fields
5820	* before using them.
5821	* @param cbInstr The value of RIP that is to be pushed on the guest
5822	* stack.
5823	*/
5824	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5825	{
5826	/* Inject the overflow exception. */
5827	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5828	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5829	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5830	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5831	}
5832
5833
5834	/**
5835	* Injects a general-protection (#GP) fault into the VM.
5836	*
5837	* @returns VBox status code (informational status code included).
5838	* @param pVM Pointer to the VM.
5839	* @param pVCpu Pointer to the VMCPU.
5840	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5841	* out-of-sync. Make sure to update the required fields
5842	* before using them.
5843	* @param u32ErrorCode The error code associated with the #GP.
5844	*/
5845	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5846	{
5847	/* Inject the general-protection fault. */
5848	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5849	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5850	if (fErrorCodeValid)
5851	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5852	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5853	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5854	}
5855
5856
5857	/**
5858	* Injects a software interrupt (INTn) into the VM.
5859	*
5860	* @returns VBox status code (informational status code included).
5861	* @param pVM Pointer to the VM.
5862	* @param pVCpu Pointer to the VMCPU.
5863	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5864	* out-of-sync. Make sure to update the required fields
5865	* before using them.
5866	* @param uVector The software interrupt vector number.
5867	* @param cbInstr The value of RIP that is to be pushed on the guest
5868	* stack.
5869	*/
5870	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5871	{
5872	/* Inject the INTn. */
5873	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5874	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5875	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5876	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5877	}
5878
5879
5880	/**
5881	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5882	* stack.
5883	*
5884	* @returns VBox status code (information status code included).
5885	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5886	* @param pVM Pointer to the VM.
5887	* @param pMixedCtx Pointer to the guest-CPU context.
5888	* @param uValue The value to push to the guest stack.
5889	*/
5890	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5891	{
5892	/*
5893	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5894	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5895	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5896	*/
5897	if (pMixedCtx->sp == 1)
5898	return VINF_EM_RESET;
5899	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5900	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5901	AssertRCReturn(rc, rc);
5902	return rc;
5903	}
5904
5905
5906	/**
5907	* Injects an event into the guest upon VM-entry by updating the relevant fields
5908	* in the VM-entry area in the VMCS.
5909	*
5910	* @returns VBox status code (informational error codes included).
5911	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5912	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5913	*
5914	* @param pVM Pointer to the VM.
5915	* @param pVCpu Pointer to the VMCPU.
5916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5917	* out-of-sync. Make sure to update the required fields
5918	* before using them.
5919	* @param u64IntrInfo The VM-entry interruption-information field.
5920	* @param cbInstr The VM-entry instruction length in bytes (for software
5921	* interrupts, exceptions and privileged software
5922	* exceptions).
5923	* @param u32ErrCode The VM-entry exception error code.
5924	*
5925	* @remarks No-long-jump zone!!!
5926	*/
5927	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5928	uint32_t u32ErrCode)
5929	{
5930	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5931	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5932	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5933
5934	/* We require CR0 to check if the guest is in real-mode. */
5935	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5936	AssertRCReturn(rc, rc);
5937
5938	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5939	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5940
5941	/*
5942	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5943	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5944	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5945	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5946	*/
5947	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5948	{
5949	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5950	{
5951	Assert(PDMVmmDevHeapIsEnabled(pVM));
5952	Assert(pVM->hm.s.vmx.pRealModeTSS);
5953
5954	/* Save the required guest state bits from the VMCS. */
5955	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5956	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5957	AssertRCReturn(rc, rc);
5958
5959	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
5960	const size_t cbIdtEntry = 4;
5961	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
5962	{
5963	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
5964	if (uVector == X86_XCPT_DF)
5965	return VINF_EM_RESET;
5966	else if (uVector == X86_XCPT_GP)
5967	{
5968	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
5969	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
5970	}
5971
5972	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
5973	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
5974	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
5975	}
5976
5977	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
5978	uint16_t uGuestIp = pMixedCtx->ip;
5979	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
5980	{
5981	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
5982	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
5983	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
5984	}
5985	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
5986	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
5987
5988	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
5989	uint16_t offIdtEntry = 0;
5990	RTSEL selIdtEntry = 0;
5991	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
5992	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
5993	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
5994	AssertRCReturn(rc, rc);
5995
5996	/* Construct the stack frame for the interrupt/exception handler. */
5997	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
5998	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
5999	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6000	AssertRCReturn(rc, rc);
6001
6002	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6003	if (rc == VINF_SUCCESS)
6004	{
6005	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6006	pMixedCtx->rip = offIdtEntry;
6007	pMixedCtx->cs.Sel = selIdtEntry;
6008	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6009
6010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6011	\| HM_CHANGED_GUEST_RIP
6012	\| HM_CHANGED_GUEST_RFLAGS
6013	\| HM_CHANGED_GUEST_RSP;
6014	AssertRCReturn(rc, rc);
6015	}
6016	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6017	return rc;
6018	}
6019	else
6020	{
6021	/*
6022	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6023	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6024	*/
6025	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6026	}
6027	}
6028
6029	/* Add the valid bit, maybe the caller was lazy. */
6030	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6031
6032	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6033	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6034	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6035
6036	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6037	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6038	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6039	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6040	AssertRCReturn(rc, rc);
6041
6042	return rc;
6043	}
6044
6045
6046	/**
6047	* Enters the VT-x session.
6048	*
6049	* @returns VBox status code.
6050	* @param pVM Pointer to the VM.
6051	* @param pVCpu Pointer to the VMCPU.
6052	* @param pCpu Pointer to the CPU info struct.
6053	*/
6054	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6055	{
6056	AssertPtr(pVM);
6057	AssertPtr(pVCpu);
6058	Assert(pVM->hm.s.vmx.fSupported);
6059	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6060	NOREF(pCpu);
6061
6062	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6063
6064	/* Make sure we're in VMX root mode. */
6065	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6066	if (!(u32HostCR4 & X86_CR4_VMXE))
6067	{
6068	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6069	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6070	}
6071
6072	/* Load the active VMCS as the current one. */
6073	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6074	if (RT_FAILURE(rc))
6075	return rc;
6076
6077	/** @todo this will change with preemption hooks where can can VMRESUME as long
6078	* as we're no preempted. */
6079	pVCpu->hm.s.fResumeVM = false;
6080	return VINF_SUCCESS;
6081	}
6082
6083
6084	/**
6085	* Leaves the VT-x session.
6086	*
6087	* @returns VBox status code.
6088	* @param pVM Pointer to the VM.
6089	* @param pVCpu Pointer to the VMCPU.
6090	* @param pCtx Pointer to the guest-CPU context.
6091	*/
6092	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6093	{
6094	AssertPtr(pVCpu);
6095	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6096	NOREF(pVM);
6097	NOREF(pCtx);
6098
6099	/** @todo this will change with preemption hooks where we only VMCLEAR when
6100	* we are actually going to be preempted, not all the time like we
6101	* currently do. */
6102	/*
6103	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6104	* and mark the VMCS launch-state as "clear".
6105	*/
6106	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6107	return rc;
6108	}
6109
6110
6111	/**
6112	* Saves the host state in the VMCS host-state.
6113	* Sets up the VM-exit MSR-load area.
6114	*
6115	* The CPU state will be loaded from these fields on every successful VM-exit.
6116	*
6117	* @returns VBox status code.
6118	* @param pVM Pointer to the VM.
6119	* @param pVCpu Pointer to the VMCPU.
6120	*
6121	* @remarks No-long-jump zone!!!
6122	*/
6123	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6124	{
6125	AssertPtr(pVM);
6126	AssertPtr(pVCpu);
6127	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6128
6129	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6130
6131	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6132	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6133	return VINF_SUCCESS;
6134
6135	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6136	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6137
6138	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6139	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6140
6141	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6142	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6143
6144	/* Reset the host-state-changed flag. */
6145	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6146	return rc;
6147	}
6148
6149
6150	/**
6151	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6152	* loaded from these fields on every successful VM-entry.
6153	*
6154	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6155	* Sets up the VM-entry controls.
6156	* Sets up the appropriate VMX non-root function to execute guest code based on
6157	* the guest CPU mode.
6158	*
6159	* @returns VBox status code.
6160	* @param pVM Pointer to the VM.
6161	* @param pVCpu Pointer to the VMCPU.
6162	* @param pCtx Pointer to the guest-CPU context.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6167	{
6168	AssertPtr(pVM);
6169	AssertPtr(pVCpu);
6170	AssertPtr(pCtx);
6171	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6172
6173	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6174
6175	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6176
6177	/* Determine real-on-v86 mode. */
6178	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6179	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6180	&& CPUMIsGuestInRealModeEx(pCtx))
6181	{
6182	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6183	}
6184
6185	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6186	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6187
6188	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6189	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6190
6191	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6192	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6193
6194	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6195	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6196
6197	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6198	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6199
6200	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6201	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6202
6203	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6204	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6205
6206	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6207	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6208
6209	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6210	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6211
6212	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6213	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6214
6215	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6216	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6217	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6218
6219	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6220	return rc;
6221	}
6222
6223
6224	/**
6225	* Does the preparations before executing guest code in VT-x.
6226	*
6227	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6228	* recompiler. We must be cautious what we do here regarding committing
6229	* guest-state information into the the VMCS assuming we assuredly execute the
6230	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6231	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6232	* that the recompiler can (and should) use them when it resumes guest
6233	* execution. Otherwise such operations must be done when we can no longer
6234	* exit to ring-3.
6235	*
6236	* @returns VBox status code (informational status codes included).
6237	* @retval VINF_SUCCESS if we can proceed with running the guest.
6238	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6239	* into the guest.
6240	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6241	*
6242	* @param pVM Pointer to the VM.
6243	* @param pVCpu Pointer to the VMCPU.
6244	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6245	* out-of-sync. Make sure to update the required fields
6246	* before using them.
6247	* @param pVmxTransient Pointer to the VMX transient structure.
6248	*
6249	* @remarks Called with preemption disabled.
6250	*/
6251	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6252	{
6253	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6254
6255	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6256	PGMRZDynMapFlushAutoSet(pVCpu);
6257	#endif
6258
6259	/* Check force flag actions that might require us to go back to ring-3. */
6260	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6261	if (rc != VINF_SUCCESS)
6262	return rc;
6263
6264	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6265	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6266	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6267	{
6268	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6269	RTGCPHYS GCPhysApicBase;
6270	GCPhysApicBase = pMixedCtx->msrApicBase;
6271	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6272
6273	/* Unalias any existing mapping. */
6274	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6275	AssertRCReturn(rc, rc);
6276
6277	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6278	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6279	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6280	AssertRCReturn(rc, rc);
6281
6282	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6283	}
6284
6285	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6286	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6287	pVmxTransient->uEFlags = ASMIntDisableFlags();
6288	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6289	{
6290	ASMSetFlags(pVmxTransient->uEFlags);
6291	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6292	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6293	return VINF_EM_RAW_INTERRUPT;
6294	}
6295	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6296	#endif
6297
6298	/*
6299	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6300	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6301	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6302	*/
6303	hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6304	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6305	AssertRCReturn(rc, rc);
6306	return rc;
6307	}
6308
6309
6310	/**
6311	* Prepares to run guest code in VT-x and we've committed to doing so. This
6312	* means there is no backing out to ring-3 or anywhere else at this
6313	* point.
6314	*
6315	* @param pVM Pointer to the VM.
6316	* @param pVCpu Pointer to the VMCPU.
6317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6318	* out-of-sync. Make sure to update the required fields
6319	* before using them.
6320	* @param pVmxTransient Pointer to the VMX transient structure.
6321	*
6322	* @remarks Called with preemption disabled.
6323	* @remarks No-long-jump zone!!!
6324	*/
6325	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6326	{
6327	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6328	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6329
6330	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6331	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6332	pVmxTransient->uEFlags = ASMIntDisableFlags();
6333	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6334	#endif
6335
6336	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6337	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6338	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6339	int rc = VINF_SUCCESS;
6340	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6341	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6342	else if (pVCpu->hm.s.fContextUseFlags)
6343	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6344	AssertRC(rc);
6345	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6346
6347	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6348	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6349	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6350
6351	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6352	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6353	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6354
6355	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6356	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6357	{
6358	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6359	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6360	}
6361
6362	/*
6363	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6364	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6365	*/
6366	if (pVM->hm.s.fTPRPatchingActive)
6367	{
6368	Assert(!CPUMIsGuestInLongMode(pVCpu));
6369
6370	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6371	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6372	AssertRC(rc);
6373
6374	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6375	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6376	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6377	see hmR0VmxLoadGuestApicState(). */
6378	}
6379
6380	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6381	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6382	to start executing. */
6383	}
6384
6385
6386	/**
6387	* Performs some essential restoration of state after running guest code in
6388	* VT-x.
6389	*
6390	* @param pVM Pointer to the VM.
6391	* @param pVCpu Pointer to the VMCPU.
6392	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6393	* out-of-sync. Make sure to update the required fields
6394	* before using them.
6395	* @param pVmxTransient Pointer to the VMX transient structure.
6396	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6397	*
6398	* @remarks Called with interrupts disabled.
6399	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6400	* unconditionally when it is safe to do so.
6401	*/
6402	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6403	{
6404	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6405	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6406
6407	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6408	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6409	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6410	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6411	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6412
6413	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6414	{
6415	/** @todo Find a way to fix hardcoding a guestimate. */
6416	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6417	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6418	}
6419
6420	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6421	Assert(!(ASMGetFlags() & X86_EFL_IF));
6422	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6423
6424	/* Restore the effects of TPR patching if any. */
6425	if (pVM->hm.s.fTPRPatchingActive)
6426	{
6427	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6428	AssertRC(rc);
6429	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6430	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6431	}
6432
6433	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6434	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6435
6436	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6437	uint32_t uExitReason;
6438	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6439	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6440	AssertRC(rc);
6441	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6442	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6443
6444	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6445	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6446
6447	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6448	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6449	{
6450	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6451	return;
6452	}
6453
6454	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6455	{
6456	/* Update the guest interruptibility-state from the VMCS. */
6457	hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6458
6459	/*
6460	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6461	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6462	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6463	*/
6464	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6465	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6466	{
6467	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6468	AssertRC(rc);
6469	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6470	}
6471	}
6472	}
6473
6474
6475	/**
6476	* Runs the guest code using VT-x.
6477	*
6478	* @returns VBox status code.
6479	* @param pVM Pointer to the VM.
6480	* @param pVCpu Pointer to the VMCPU.
6481	* @param pCtx Pointer to the guest-CPU context.
6482	*
6483	* @remarks Called with preemption disabled.
6484	*/
6485	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6486	{
6487	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6488	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6489
6490	VMXTRANSIENT VmxTransient;
6491	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6492	int rc = VERR_INTERNAL_ERROR_5;
6493	unsigned cLoops = 0;
6494
6495	for (;; cLoops++)
6496	{
6497	Assert(!HMR0SuspendPending());
6498	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6499	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6500	(unsigned)RTMpCpuId(), cLoops));
6501
6502	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6503	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6504	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6505	if (rc != VINF_SUCCESS)
6506	break;
6507
6508	/*
6509	* No longjmps to ring-3 from this point on!!!
6510	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6511	* This also disables flushing of the R0-logger instance (if any).
6512	*/
6513	VMMRZCallRing3Disable(pVCpu);
6514	VMMRZCallRing3RemoveNotification(pVCpu);
6515	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6516
6517	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6518	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6519
6520	/*
6521	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6522	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6523	*/
6524	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6525	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6526	{
6527	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6528	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6529	return rc;
6530	}
6531
6532	/* Handle the VM-exit. */
6533	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6534	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6535	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6536	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6537	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6538	if (rc != VINF_SUCCESS)
6539	break;
6540	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6541	{
6542	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6543	rc = VINF_EM_RAW_INTERRUPT;
6544	break;
6545	}
6546	}
6547
6548	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6549	if (rc == VERR_EM_INTERPRETER)
6550	rc = VINF_EM_RAW_EMULATE_INSTR;
6551	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6552	return rc;
6553	}
6554
6555	#ifdef DEBUG
6556	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6557	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6558	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6559	# define VMX_ASSERT_PREEMPT_CPUID() \
6560	do \
6561	{ \
6562	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6563	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6564	} while (0)
6565
6566	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6567	do { \
6568	AssertPtr(pVM); \
6569	AssertPtr(pVCpu); \
6570	AssertPtr(pMixedCtx); \
6571	AssertPtr(pVmxTransient); \
6572	Assert(pVmxTransient->fVMEntryFailed == false); \
6573	Assert(ASMIntAreEnabled() == true); \
6574	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6575	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6576	LogFunc(("\n")); \
6577	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6578	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6579	VMX_ASSERT_PREEMPT_CPUID(); \
6580	} while (0)
6581	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6582	do { \
6583	LogFunc(("\n")); \
6584	} while(0)
6585	#else /* Release builds */
6586	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6587	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6588	#endif
6589
6590
6591	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6592	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6593	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6594	/**
6595	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6596	*/
6597	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6598	{
6599	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6601	return VINF_SUCCESS;
6602	}
6603
6604
6605	/**
6606	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6607	*/
6608	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6609	{
6610	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6611	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6612	AssertRCReturn(rc, rc);
6613
6614	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6615	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6616	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6617
6618	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6619	return VINF_EM_RAW_INTERRUPT;
6620
6621	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6622	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6623	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6624	return VINF_SUCCESS;
6625	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6626	return rc;
6627
6628	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6629	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6630	switch (uIntrType)
6631	{
6632	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6633	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6634	/* no break */
6635	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6636	{
6637	switch (uVector)
6638	{
6639	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6640	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6641	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6642	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6643	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6644	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6645	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6646	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6647	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6648	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6649	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6650	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6651	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6652	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6653	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6654	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6655	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6656	#endif
6657	default:
6658	{
6659	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6660	AssertRCReturn(rc, rc);
6661
6662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6663	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6664	{
6665	Assert(pVM->hm.s.vmx.pRealModeTSS);
6666	Assert(PDMVmmDevHeapIsEnabled(pVM));
6667	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6668	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6669	AssertRCReturn(rc, rc);
6670	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6671	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6672	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6673	AssertRCReturn(rc, rc);
6674	}
6675	else
6676	{
6677	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6678	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6679	}
6680	break;
6681	}
6682	}
6683	break;
6684	}
6685
6686	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6687	default:
6688	{
6689	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6690	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6691	break;
6692	}
6693	}
6694	return rc;
6695	}
6696
6697
6698	/**
6699	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6700	*/
6701	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6702	{
6703	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6704
6705	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6706	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6707	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6708	AssertRCReturn(rc, rc);
6709
6710	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6712	return VINF_SUCCESS;
6713	}
6714
6715
6716	/**
6717	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6718	*/
6719	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6720	{
6721	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6722	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6723	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6724	}
6725
6726
6727	/**
6728	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6729	*/
6730	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6731	{
6732	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6733	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6734	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6735	AssertRCReturn(rc, rc);
6736
6737	pMixedCtx->rip += pVmxTransient->cbInstr;
6738	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6739
6740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6741	return VINF_SUCCESS;
6742	}
6743
6744
6745	/**
6746	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6747	*/
6748	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6749	{
6750	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6751	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6752	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6753	AssertRCReturn(rc, rc);
6754
6755	pMixedCtx->rip += pVmxTransient->cbInstr;
6756	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6757
6758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6759	return VINF_SUCCESS;
6760	}
6761
6762
6763	/**
6764	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6765	*/
6766	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6767	{
6768	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6769	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6770	if (RT_LIKELY(rc == VINF_SUCCESS))
6771	{
6772	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6773	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6774	AssertRCReturn(rc, rc);
6775	Assert(pVmxTransient->cbInstr == 2);
6776
6777	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6778	pMixedCtx->rip += pVmxTransient->cbInstr;
6779	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6780	}
6781	else
6782	{
6783	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6784	rc = VERR_EM_INTERPRETER;
6785	}
6786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6787	return rc;
6788	}
6789
6790
6791	/**
6792	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6793	*/
6794	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6795	{
6796	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6797	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6798	AssertRCReturn(rc, rc);
6799
6800	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6801	return VINF_EM_RAW_EMULATE_INSTR;
6802
6803	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6804	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6805	}
6806
6807
6808	/**
6809	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6810	*/
6811	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6812	{
6813	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6814	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6815	AssertRCReturn(rc, rc);
6816
6817	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6818	if (RT_LIKELY(rc == VINF_SUCCESS))
6819	{
6820	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6821	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6822	AssertRCReturn(rc, rc);
6823	Assert(pVmxTransient->cbInstr == 2);
6824
6825	pMixedCtx->rip += pVmxTransient->cbInstr;
6826	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6827
6828	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6829	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6830	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
6831	}
6832	else
6833	{
6834	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6835	rc = VERR_EM_INTERPRETER;
6836	}
6837	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6838	return rc;
6839	}
6840
6841
6842	/**
6843	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6844	*/
6845	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6846	{
6847	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6848	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6849	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6850	AssertRCReturn(rc, rc);
6851
6852	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6853	if (RT_LIKELY(rc == VINF_SUCCESS))
6854	{
6855	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6856	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6857	AssertRCReturn(rc, rc);
6858	Assert(pVmxTransient->cbInstr == 3);
6859
6860	pMixedCtx->rip += pVmxTransient->cbInstr;
6861	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6862
6863	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6864	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6865	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
6866	}
6867	else
6868	{
6869	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6870	rc = VERR_EM_INTERPRETER;
6871	}
6872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6873	return rc;
6874	}
6875
6876
6877	/**
6878	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6879	*/
6880	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6881	{
6882	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6883	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6884	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6885	AssertRCReturn(rc, rc);
6886
6887	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6888	if (RT_LIKELY(rc == VINF_SUCCESS))
6889	{
6890	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6891	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6892	AssertRCReturn(rc, rc);
6893	Assert(pVmxTransient->cbInstr == 2);
6894
6895	pMixedCtx->rip += pVmxTransient->cbInstr;
6896	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6897	}
6898	else
6899	{
6900	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6901	rc = VERR_EM_INTERPRETER;
6902	}
6903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6904	return rc;
6905	}
6906
6907
6908	/**
6909	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6910	*/
6911	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6912	{
6913	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6914	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6915	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6916	AssertRCReturn(rc, rc);
6917
6918	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6919	rc = VBOXSTRICTRC_VAL(rc2);
6920	if (RT_LIKELY(rc == VINF_SUCCESS))
6921	{
6922	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6923	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6924	AssertRCReturn(rc, rc);
6925
6926	pMixedCtx->rip += pVmxTransient->cbInstr;
6927	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6928	}
6929	else
6930	{
6931	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6932	pVmxTransient->uExitQualification, rc));
6933	rc = VERR_EM_INTERPRETER;
6934	}
6935	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6936	return rc;
6937	}
6938
6939
6940	/**
6941	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6942	*/
6943	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6944	{
6945	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6946	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6947	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6948	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6949	AssertRCReturn(rc, rc);
6950
6951	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6952	if (RT_LIKELY(rc == VINF_SUCCESS))
6953	{
6954	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6955	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6956	AssertRCReturn(rc, rc);
6957
6958	pMixedCtx->rip += pVmxTransient->cbInstr;
6959	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6960	}
6961	else
6962	{
6963	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6964	rc = VERR_EM_INTERPRETER;
6965	}
6966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6967	return rc;
6968	}
6969
6970
6971	/**
6972	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
6973	*/
6974	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6975	{
6976	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6977	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6978	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6979	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6980	AssertRCReturn(rc, rc);
6981
6982	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6983	rc = VBOXSTRICTRC_VAL(rc2);
6984	if (RT_LIKELY( rc == VINF_SUCCESS
6985	\|\| rc == VINF_EM_HALT))
6986	{
6987	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6988	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6989	AssertRCReturn(rc3, rc3);
6990
6991	pMixedCtx->rip += pVmxTransient->cbInstr;
6992	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6993
6994	if ( rc == VINF_EM_HALT
6995	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
6996	{
6997	rc = VINF_SUCCESS;
6998	}
6999	}
7000	else
7001	{
7002	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7003	rc = VERR_EM_INTERPRETER;
7004	}
7005	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7006	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7007	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7008	return rc;
7009	}
7010
7011
7012	/**
7013	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7014	*/
7015	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7016	{
7017	/*
7018	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7019	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7020	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7021	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7022	*/
7023	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7024	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7025	}
7026
7027
7028	/**
7029	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7030	*/
7031	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7032	{
7033	/*
7034	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7035	* root operation. If we get there there is something funny going on.
7036	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7037	*/
7038	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7039	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7040	}
7041
7042
7043	/**
7044	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7045	*/
7046	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7047	{
7048	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7049	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7050	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7051	}
7052
7053
7054	/**
7055	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7056	*/
7057	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7058	{
7059	/*
7060	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7061	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7062	* See Intel spec. 25.3 "Other Causes of VM-exits".
7063	*/
7064	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7065	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7066	}
7067
7068
7069	/**
7070	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7071	* VM-exit.
7072	*/
7073	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7074	{
7075	/*
7076	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7077	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7078	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7079	*/
7080	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7081	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7082	}
7083
7084
7085	/**
7086	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7087	* VM-exit.
7088	*/
7089	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7090	{
7091	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7092	return VINF_EM_RESET;
7093	}
7094
7095
7096	/**
7097	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7098	*/
7099	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7100	{
7101	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7102	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7103	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7104	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7105	AssertRCReturn(rc, rc);
7106
7107	pMixedCtx->rip++;
7108	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7109	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7110	rc = VINF_SUCCESS;
7111	else
7112	rc = VINF_EM_HALT;
7113
7114	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7115	return rc;
7116	}
7117
7118
7119	/**
7120	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7121	*/
7122	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7123	{
7124	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7125	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7126	AssertRCReturn(rc, rc);
7127	return rc;
7128	}
7129
7130
7131	/**
7132	* VM-exit handler for expiry of the VMX preemption timer.
7133	*/
7134	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7135	{
7136	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7137
7138	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7139	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7140	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7141
7142	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7143	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7145	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7146	}
7147
7148
7149	/**
7150	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7151	*/
7152	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7153	{
7154	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7155	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7156	/** @todo check if XSETBV is supported by the recompiler. */
7157	return VERR_EM_INTERPRETER;
7158	}
7159
7160
7161	/**
7162	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7163	*/
7164	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7165	{
7166	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7167	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7168	/** @todo implement EMInterpretInvpcid() */
7169	return VERR_EM_INTERPRETER;
7170	}
7171
7172
7173	/**
7174	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7175	* Error VM-exit.
7176	*/
7177	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7178	{
7179	uint32_t uIntrState;
7180	RTHCUINTREG uHCReg;
7181	uint64_t u64Val;
7182
7183	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7184	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7185	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7186	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7187	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7188	AssertRCReturn(rc, rc);
7189
7190	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7191	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7192	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7193	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7194
7195	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7196	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7197	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7198	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7199	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7200	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7201	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7202	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7203	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7204	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7205	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7206	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7207
7208	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7209
7210	return VERR_VMX_INVALID_GUEST_STATE;
7211	}
7212
7213
7214	/**
7215	* VM-exit handler for VM-entry failure due to an MSR-load
7216	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7217	*/
7218	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7219	{
7220	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7221	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7222	}
7223
7224
7225	/**
7226	* VM-exit handler for VM-entry failure due to a machine-check event
7227	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7228	*/
7229	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7230	{
7231	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7232	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7233	}
7234
7235
7236	/**
7237	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7238	* theory.
7239	*/
7240	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7241	{
7242	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7243	pVM, pVCpu, pMixedCtx));
7244	return VERR_VMX_UNDEFINED_EXIT_CODE;
7245	}
7246
7247
7248	/**
7249	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7250	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7251	* Conditional VM-exit.
7252	*/
7253	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7254	{
7255	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7256	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7258	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7259	return VERR_EM_INTERPRETER;
7260	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7261	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7262	}
7263
7264
7265	/**
7266	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7267	*/
7268	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7269	{
7270	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7271	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7273	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7274	return VERR_EM_INTERPRETER;
7275	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7276	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7277	}
7278
7279
7280	/**
7281	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7282	*/
7283	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7284	{
7285	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7286	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7287	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7288	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7289	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7290	AssertRCReturn(rc, rc);
7291
7292	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7293	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7294	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7296
7297	/* Update RIP and continue guest execution. */
7298	if (RT_LIKELY(rc == VINF_SUCCESS))
7299	{
7300	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7301	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7302	AssertRCReturn(rc, rc);
7303
7304	Assert(pVmxTransient->cbInstr == 2);
7305	pMixedCtx->rip += pVmxTransient->cbInstr;
7306	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7307	}
7308	return rc;
7309	}
7310
7311
7312	/**
7313	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7314	*/
7315	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7316	{
7317	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7318	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7319	AssertRCReturn(rc, rc);
7320	Assert(pVmxTransient->cbInstr == 2);
7321
7322	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7323	if ( pVM->hm.s.fTPRPatchingActive
7324	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7325	{
7326	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7327	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7328	{
7329	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7330	AssertRC(rc);
7331	}
7332
7333	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7334	AssertRCReturn(rc, rc);
7335	pMixedCtx->rip += pVmxTransient->cbInstr;
7336	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7338	return VINF_SUCCESS;
7339	}
7340
7341	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7342	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7343	{
7344	switch (pMixedCtx->ecx)
7345	{
7346	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7347	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7348	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7349	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7350	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7351	}
7352	}
7353	#ifdef DEBUG
7354	else
7355	{
7356	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7357	switch (pMixedCtx->ecx)
7358	{
7359	case MSR_IA32_SYSENTER_CS:
7360	case MSR_IA32_SYSENTER_EIP:
7361	case MSR_IA32_SYSENTER_ESP:
7362	case MSR_K8_FS_BASE:
7363	case MSR_K8_GS_BASE:
7364	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7365	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7366	case MSR_K8_LSTAR:
7367	case MSR_K6_STAR:
7368	case MSR_K8_SF_MASK:
7369	case MSR_K8_TSC_AUX:
7370	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7371	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7372	}
7373	}
7374	#endif
7375
7376	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7377	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7378	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7379	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7380	AssertRCReturn(rc, rc);
7381
7382	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7383	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7385
7386	/* Update guest-state and continue execution. */
7387	if (RT_LIKELY(rc == VINF_SUCCESS))
7388	{
7389	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7390	AssertRCReturn(rc, rc);
7391
7392	pMixedCtx->rip += pVmxTransient->cbInstr;
7393	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7394
7395	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7396	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7397	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7398	{
7399	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7400	}
7401	}
7402	return rc;
7403	}
7404
7405
7406	/**
7407	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7408	*/
7409	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7410	{
7411	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7412	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7413	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7414	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7415	return VERR_EM_INTERPRETER;
7416	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7417	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7418	}
7419
7420
7421	/**
7422	* VM-exit handler for when the TPR value is lowered below the specified
7423	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7424	*/
7425	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7426	{
7427	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7428	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7429
7430	/*
7431	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7432	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7433	* resume guest execution.
7434	*/
7435	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7436	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7437	return VINF_SUCCESS;
7438	}
7439
7440
7441	/**
7442	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7443	* VM-exit.
7444	*
7445	* @retval VINF_SUCCESS when guest execution can continue.
7446	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7447	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7448	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7449	* recompiler.
7450	*/
7451	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7452	{
7453	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7454	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7455	AssertRCReturn(rc, rc);
7456
7457	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7458	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7459	switch (uAccessType)
7460	{
7461	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7462	{
7463	#if 0
7464	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7465	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7466	#else
7467	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7468	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7469	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7470	#endif
7471	AssertRCReturn(rc, rc);
7472
7473	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7474	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7475	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7476	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7477
7478	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7479	{
7480	case 0: /* CR0 */
7481	Log(("CR0 write rc=%d\n", rc));
7482	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7483	break;
7484	case 2: /* CR2 */
7485	Log(("CR2 write rc=%d\n", rc));
7486	break;
7487	case 3: /* CR3 */
7488	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7489	Log(("CR3 write rc=%d\n", rc));
7490	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7491	break;
7492	case 4: /* CR4 */
7493	Log(("CR4 write rc=%d\n", rc));
7494	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7495	break;
7496	case 8: /* CR8 */
7497	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7498	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7499	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7500	break;
7501	default:
7502	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7503	break;
7504	}
7505
7506	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7507	break;
7508	}
7509
7510	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7511	{
7512	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7513	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7514	AssertRCReturn(rc, rc);
7515	Assert( !pVM->hm.s.fNestedPaging
7516	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7517	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7518
7519	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7520	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7521	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7522
7523	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7524	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7525	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7526	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7527	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7528	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7529	break;
7530	}
7531
7532	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7533	{
7534	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7535	AssertRCReturn(rc, rc);
7536	rc = EMInterpretCLTS(pVM, pVCpu);
7537	AssertRCReturn(rc, rc);
7538	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7540	Log(("CRX CLTS write rc=%d\n", rc));
7541	break;
7542	}
7543
7544	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7545	{
7546	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7547	AssertRCReturn(rc, rc);
7548	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7549	if (RT_LIKELY(rc == VINF_SUCCESS))
7550	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7551	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7552	Log(("CRX LMSW write rc=%d\n", rc));
7553	break;
7554	}
7555
7556	default:
7557	{
7558	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7559	return VERR_VMX_UNEXPECTED_EXCEPTION;
7560	}
7561	}
7562
7563	/* Validate possible error codes. */
7564	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7565	if (RT_SUCCESS(rc))
7566	{
7567	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7568	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7569	AssertRCReturn(rc2, rc2);
7570	pMixedCtx->rip += pVmxTransient->cbInstr;
7571	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7572	}
7573
7574	return rc;
7575	}
7576
7577
7578	/**
7579	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7580	* VM-exit.
7581	*/
7582	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7583	{
7584	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7585
7586	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7587	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7588	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7589	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7590	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7591	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7592	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7593	AssertRCReturn(rc, rc);
7594
7595	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7596	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7597	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7598	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7599	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7600	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7601	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7602
7603	/* I/O operation lookup arrays. */
7604	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7605	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7606
7607	const uint32_t cbSize = s_aIOSize[uIOWidth];
7608	const uint32_t cbInstr = pVmxTransient->cbInstr;
7609	if (fIOString)
7610	{
7611	/* INS/OUTS - I/O String instruction. */
7612	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7613	/** @todo for now manually disassemble later optimize by getting the fields from
7614	* the VMCS. */
7615	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7616	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7617	* segment prefix info. */
7618	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7619	if (RT_SUCCESS(rc))
7620	{
7621	if (fIOWrite)
7622	{
7623	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7624	(DISCPUMODE)pDis->uAddrMode, cbSize);
7625	rc = VBOXSTRICTRC_VAL(rc2);
7626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7627	}
7628	else
7629	{
7630	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7631	(DISCPUMODE)pDis->uAddrMode, cbSize);
7632	rc = VBOXSTRICTRC_VAL(rc2);
7633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7634	}
7635	}
7636	else
7637	{
7638	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7639	rc = VINF_EM_RAW_EMULATE_INSTR;
7640	}
7641	}
7642	else
7643	{
7644	/* IN/OUT - I/O instruction. */
7645	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7646	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7647	if (fIOWrite)
7648	{
7649	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7650	rc = VBOXSTRICTRC_VAL(rc2);
7651	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7652	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7653	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7654	}
7655	else
7656	{
7657	uint32_t u32Result = 0;
7658	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7659	rc = VBOXSTRICTRC_VAL(rc2);
7660	if (IOM_SUCCESS(rc))
7661	{
7662	/* Save result of I/O IN instr. in AL/AX/EAX. */
7663	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7664	}
7665	else if (rc == VINF_IOM_R3_IOPORT_READ)
7666	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7668	}
7669	}
7670
7671	if (IOM_SUCCESS(rc))
7672	{
7673	pMixedCtx->rip += cbInstr;
7674	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7675	if (RT_LIKELY(rc == VINF_SUCCESS))
7676	{
7677	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7678	AssertRCReturn(rc, rc);
7679
7680	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7681	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7682	{
7683	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7684	for (unsigned i = 0; i < 4; i++)
7685	{
7686	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7687	if ( ( uIOPort >= pMixedCtx->dr[i]
7688	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7689	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7690	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7691	{
7692	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7693	uint64_t uDR6 = ASMGetDR6();
7694
7695	/* Clear all breakpoint status flags and set the one we just hit. */
7696	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7697	uDR6 \|= (uint64_t)RT_BIT(i);
7698
7699	/*
7700	* Note: AMD64 Architecture Programmer's Manual 13.1:
7701	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7702	* be cleared by software after the contents have been read.
7703	*/
7704	ASMSetDR6(uDR6);
7705
7706	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7707	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7708
7709	/* Paranoia. */
7710	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7711	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7712	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7713
7714	/* Resync DR7 */
7715	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7716	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7717
7718	/* Inject #DB and get on with guest execution. */
7719	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7720	AssertRCReturn(rc, rc);
7721	break;
7722	}
7723	}
7724	}
7725	}
7726	}
7727
7728	#ifdef DEBUG
7729	if (rc == VINF_IOM_R3_IOPORT_READ)
7730	Assert(!fIOWrite);
7731	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7732	Assert(fIOWrite);
7733	else
7734	{
7735	AssertMsg( RT_FAILURE(rc)
7736	\|\| rc == VINF_SUCCESS
7737	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7738	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7739	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7740	}
7741	#endif
7742
7743	return rc;
7744	}
7745
7746
7747	/**
7748	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7749	* VM-exit.
7750	*/
7751	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7752	{
7753	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7754
7755	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7756	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7757	AssertRCReturn(rc, rc);
7758	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7759	{
7760	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7761	AssertRCReturn(rc, rc);
7762	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7763	{
7764	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7765	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7766	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7767	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7768	{
7769	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7770	pVCpu->hm.s.Event.fPending = true;
7771	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7772	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7773	AssertRCReturn(rc, rc);
7774	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7775	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7776	else
7777	pVCpu->hm.s.Event.u32ErrCode = 0;
7778	}
7779	}
7780	}
7781	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7782	* emulation. */
7783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7784	return VERR_EM_INTERPRETER;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7790	*/
7791	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7792	{
7793	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7794	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7795	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7796	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7797	AssertRCReturn(rc, rc);
7798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7799	return VINF_EM_DBG_STOP;
7800	}
7801
7802
7803	/**
7804	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7805	*/
7806	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7807	{
7808	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7809	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7810
7811	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7812	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7813	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7814	return VINF_SUCCESS;
7815	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7816	return rc;
7817
7818	#if 0
7819	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7820	* just sync the whole thing. */
7821	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7822	#else
7823	/* Aggressive state sync. for now. */
7824	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7825	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7826	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7827	#endif
7828	AssertRCReturn(rc, rc);
7829
7830	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7831	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7832	switch (uAccessType)
7833	{
7834	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7835	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7836	{
7837	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7838	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7839	{
7840	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7841	}
7842
7843	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7844	GCPhys &= PAGE_BASE_GC_MASK;
7845	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7846	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7847	CPUMCTX2CORE(pMixedCtx), GCPhys);
7848	rc = VBOXSTRICTRC_VAL(rc2);
7849	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7850	if ( rc == VINF_SUCCESS
7851	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7852	\|\| rc == VERR_PAGE_NOT_PRESENT)
7853	{
7854	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7855	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7856	rc = VINF_SUCCESS;
7857	}
7858	break;
7859	}
7860
7861	default:
7862	rc = VINF_EM_RAW_EMULATE_INSTR;
7863	break;
7864	}
7865
7866	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7867	return rc;
7868	}
7869
7870
7871	/**
7872	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7873	* VM-exit.
7874	*/
7875	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7876	{
7877	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7878
7879	/* We should -not- get this VM-exit if the guest is debugging. */
7880	if (CPUMIsGuestDebugStateActive(pVCpu))
7881	{
7882	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7883	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7884	}
7885
7886	int rc = VERR_INTERNAL_ERROR_5;
7887	if ( !DBGFIsStepping(pVCpu)
7888	&& !CPUMIsHyperDebugStateActive(pVCpu))
7889	{
7890	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7891
7892	/* Don't intercept MOV DRx. */
7893	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7894	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7895	AssertRCReturn(rc, rc);
7896
7897	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7898	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7899	AssertRCReturn(rc, rc);
7900	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7901
7902	#ifdef VBOX_WITH_STATISTICS
7903	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7904	AssertRCReturn(rc, rc);
7905	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7907	else
7908	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7909	#endif
7910	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7911	return VINF_SUCCESS;
7912	}
7913
7914	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7915	* time and restore DRx registers afterwards */
7916	/*
7917	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7918	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7919	*/
7920	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7921	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7922	AssertRCReturn(rc, rc);
7923
7924	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7925	{
7926	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7927	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7928	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7929	if (RT_SUCCESS(rc))
7930	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7931	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7932	}
7933	else
7934	{
7935	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7936	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7937	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7938	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7939	}
7940
7941	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7942	if (RT_SUCCESS(rc))
7943	{
7944	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7945	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7946	AssertRCReturn(rc2, rc2);
7947	pMixedCtx->rip += pVmxTransient->cbInstr;
7948	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7949	}
7950	return rc;
7951	}
7952
7953
7954	/**
7955	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7956	* Conditional VM-exit.
7957	*/
7958	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7959	{
7960	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7961	Assert(pVM->hm.s.fNestedPaging);
7962
7963	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7964	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7965	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7966	return VINF_SUCCESS;
7967	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7968	return rc;
7969
7970	RTGCPHYS GCPhys = 0;
7971	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
7972	AssertRCReturn(rc, rc);
7973
7974	#if 0
7975	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
7976	#else
7977	/* Aggressive state sync. for now. */
7978	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7979	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7980	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7981	#endif
7982	AssertRCReturn(rc, rc);
7983
7984	/*
7985	* If we succeed, resume guest execution.
7986	* If we fail in interpreting the instruction because we couldn't get the guest physical address
7987	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
7988	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
7989	* weird case. See @bugref{6043}.
7990	*/
7991	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
7992	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
7993	rc = VBOXSTRICTRC_VAL(rc2);
7994	if ( rc == VINF_SUCCESS
7995	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7996	\|\| rc == VERR_PAGE_NOT_PRESENT)
7997	{
7998	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7999	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8000	return VINF_SUCCESS;
8001	}
8002	return rc;
8003	}
8004
8005
8006	/**
8007	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8008	* VM-exit.
8009	*/
8010	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8011	{
8012	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8013	Assert(pVM->hm.s.fNestedPaging);
8014
8015	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8016	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8017	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8018	return VINF_SUCCESS;
8019	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8020	return rc;
8021
8022	RTGCPHYS GCPhys = 0;
8023	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8024	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8025	#if 0
8026	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8027	#else
8028	/* Aggressive state sync. for now. */
8029	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8030	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8031	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8032	#endif
8033	AssertRCReturn(rc, rc);
8034
8035	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8036	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8037
8038	RTGCUINT uErrorCode = 0;
8039	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8040	uErrorCode \|= X86_TRAP_PF_ID;
8041	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8042	uErrorCode \|= X86_TRAP_PF_RW;
8043	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8044	uErrorCode \|= X86_TRAP_PF_P;
8045
8046	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8047	TRPMSetErrorCode(pVCpu, uErrorCode);
8048	TRPMSetFaultAddress(pVCpu, GCPhys);
8049
8050	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8051	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8052
8053	/* Handle the pagefault trap for the nested shadow table. */
8054	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8055	TRPMResetTrap(pVCpu);
8056
8057	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8058	if ( rc == VINF_SUCCESS
8059	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8060	\|\| rc == VERR_PAGE_NOT_PRESENT)
8061	{
8062	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8064	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8065	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8066	return VINF_SUCCESS;
8067	}
8068
8069	Log(("EPT return to ring-3 rc=%d\n"));
8070
8071	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8072	/** @todo Shouldn't we update TRPM here? */
8073	return rc;
8074	}
8075
8076
8077	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8078	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8079	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8080	/**
8081	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8082	*/
8083	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8084	{
8085	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8086
8087	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8088	AssertRCReturn(rc, rc);
8089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8090
8091	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8092	{
8093	/* Old-style FPU error reporting needs some extra work. */
8094	/** @todo don't fall back to the recompiler, but do it manually. */
8095	return VERR_EM_INTERPRETER;
8096	}
8097	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8098	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8099	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8100	AssertRCReturn(rc, rc);
8101	return rc;
8102	}
8103
8104
8105	/**
8106	* VM-exit exception handler for #BP (Breakpoint exception).
8107	*/
8108	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8109	{
8110	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8111
8112	/** @todo Try optimize this by not saving the entire guest state unless
8113	* really needed. */
8114	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8115	AssertRCReturn(rc, rc);
8116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8117
8118	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8119	if (rc == VINF_EM_RAW_GUEST_TRAP)
8120	{
8121	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8122	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8123	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8124	AssertRCReturn(rc, rc);
8125
8126	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8127	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8128	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8129	AssertRCReturn(rc, rc);
8130	}
8131
8132	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8133	return rc;
8134	}
8135
8136
8137	/**
8138	* VM-exit exception handler for #DB (Debug exception).
8139	*/
8140	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8141	{
8142	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8143
8144	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8145	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8146	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8147	AssertRCReturn(rc, rc);
8148
8149	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8150	uint64_t uDR6 = X86_DR6_INIT_VAL;
8151	uDR6 \|= (pVmxTransient->uExitQualification
8152	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8153	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8155	if (rc == VINF_EM_RAW_GUEST_TRAP)
8156	{
8157	/** @todo revisit this. */
8158	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8159	pMixedCtx->dr[6] = uDR6;
8160
8161	if (CPUMIsGuestDebugStateActive(pVCpu))
8162	ASMSetDR6(pMixedCtx->dr[6]);
8163
8164	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8165	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8166
8167	/* Paranoia. */
8168	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8169	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8170	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8171
8172	/* Resync DR7. */
8173	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8174
8175	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8176	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8177	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8178	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8179	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8180	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8181	AssertRCReturn(rc,rc);
8182	return rc;
8183	}
8184	/* Return to ring 3 to deal with the debug exit code. */
8185	return rc;
8186	}
8187
8188
8189	/**
8190	* VM-exit exception handler for #NM (Device-not-available exception: floating
8191	* point exception).
8192	*/
8193	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8194	{
8195	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8196
8197	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8198	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8199	#endif
8200
8201	/* We require CR0 and EFER. EFER is always up-to-date. */
8202	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8203	AssertRCReturn(rc, rc);
8204
8205	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8206	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8207	if (rc == VINF_SUCCESS)
8208	{
8209	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8210	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8211	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8212	return VINF_SUCCESS;
8213	}
8214
8215	/* Forward #NM to the guest. */
8216	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8217	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8218	AssertRCReturn(rc, rc);
8219	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8220	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8221	pVmxTransient->cbInstr, 0 /* error code */);
8222	AssertRCReturn(rc, rc);
8223	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8224	return rc;
8225	}
8226
8227
8228	/**
8229	* VM-exit exception handler for #GP (General-protection exception).
8230	*
8231	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8232	*/
8233	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8234	{
8235	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8236
8237	int rc = VERR_INTERNAL_ERROR_5;
8238
8239	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8240	{
8241	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8242	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8243	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8244	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8245	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8246	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8247	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8248	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8249	AssertRCReturn(rc, rc);
8250	return rc;
8251	#else
8252	/* We don't intercept #GP. */
8253	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8254	return VERR_VMX_UNEXPECTED_EXCEPTION;
8255	#endif
8256	}
8257
8258	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8259	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8260
8261	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8262	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8263	AssertRCReturn(rc, rc);
8264
8265	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8266	unsigned int cbOp = 0;
8267	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8268	if (RT_SUCCESS(rc))
8269	{
8270	rc = VINF_SUCCESS;
8271	Assert(cbOp == pDis->cbInstr);
8272	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8273	switch (pDis->pCurInstr->uOpcode)
8274	{
8275	case OP_CLI:
8276	pMixedCtx->eflags.Bits.u1IF = 0;
8277	pMixedCtx->rip += pDis->cbInstr;
8278	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8280	break;
8281
8282	case OP_STI:
8283	pMixedCtx->eflags.Bits.u1IF = 1;
8284	pMixedCtx->rip += pDis->cbInstr;
8285	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8286	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8287	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8288	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8289	break;
8290
8291	case OP_HLT:
8292	rc = VINF_EM_HALT;
8293	pMixedCtx->rip += pDis->cbInstr;
8294	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8296	break;
8297
8298	case OP_POPF:
8299	{
8300	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8301	size_t cbParm = 0;
8302	uint32_t uMask = 0;
8303	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8304	{
8305	cbParm = 4;
8306	uMask = 0xffffffff;
8307	}
8308	else
8309	{
8310	cbParm = 2;
8311	uMask = 0xffff;
8312	}
8313
8314	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8315	RTGCPTR GCPtrStack = 0;
8316	X86EFLAGS uEflags;
8317	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8318	&GCPtrStack);
8319	if (RT_SUCCESS(rc))
8320	{
8321	Assert(sizeof(uEflags.u32) >= cbParm);
8322	uEflags.u32 = 0;
8323	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8324	}
8325	if (RT_FAILURE(rc))
8326	{
8327	rc = VERR_EM_INTERPRETER;
8328	break;
8329	}
8330	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8331	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8332	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8333	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8334	pMixedCtx->eflags.Bits.u1RF = 0;
8335	pMixedCtx->esp += cbParm;
8336	pMixedCtx->esp &= uMask;
8337	pMixedCtx->rip += pDis->cbInstr;
8338	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8340	break;
8341	}
8342
8343	case OP_PUSHF:
8344	{
8345	size_t cbParm = 0;
8346	uint32_t uMask = 0;
8347	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8348	{
8349	cbParm = 4;
8350	uMask = 0xffffffff;
8351	}
8352	else
8353	{
8354	cbParm = 2;
8355	uMask = 0xffff;
8356	}
8357
8358	/* Get the stack pointer & push the contents of eflags onto the stack. */
8359	RTGCPTR GCPtrStack = 0;
8360	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8361	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8362	if (RT_FAILURE(rc))
8363	{
8364	rc = VERR_EM_INTERPRETER;
8365	break;
8366	}
8367	X86EFLAGS uEflags;
8368	uEflags = pMixedCtx->eflags;
8369	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8370	uEflags.Bits.u1RF = 0;
8371	uEflags.Bits.u1VM = 0;
8372
8373	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8374	if (RT_FAILURE(rc))
8375	{
8376	rc = VERR_EM_INTERPRETER;
8377	break;
8378	}
8379	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8380	pMixedCtx->esp -= cbParm;
8381	pMixedCtx->esp &= uMask;
8382	pMixedCtx->rip += pDis->cbInstr;
8383	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8385	break;
8386	}
8387
8388	case OP_IRET:
8389	{
8390	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8391	* instruction reference. */
8392	RTGCPTR GCPtrStack = 0;
8393	uint32_t uMask = 0xffff;
8394	uint16_t aIretFrame[3];
8395	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8396	{
8397	rc = VERR_EM_INTERPRETER;
8398	break;
8399	}
8400	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8401	&GCPtrStack);
8402	if (RT_SUCCESS(rc))
8403	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8404	if (RT_FAILURE(rc))
8405	{
8406	rc = VERR_EM_INTERPRETER;
8407	break;
8408	}
8409	pMixedCtx->eip = 0;
8410	pMixedCtx->ip = aIretFrame[0];
8411	pMixedCtx->cs.Sel = aIretFrame[1];
8412	pMixedCtx->cs.ValidSel = aIretFrame[1];
8413	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8414	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8415	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8416	pMixedCtx->sp += sizeof(aIretFrame);
8417	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8418	\| HM_CHANGED_GUEST_RFLAGS;
8419	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8420	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8421	break;
8422	}
8423
8424	case OP_INT:
8425	{
8426	uint16_t uVector = pDis->Param1.uValue & 0xff;
8427	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8428	AssertRCReturn(rc, rc);
8429	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8430	break;
8431	}
8432
8433	case OP_INTO:
8434	{
8435	if (pMixedCtx->eflags.Bits.u1OF)
8436	{
8437	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8438	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8439	}
8440	break;
8441	}
8442
8443	default:
8444	{
8445	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8446	EMCODETYPE_SUPERVISOR);
8447	rc = VBOXSTRICTRC_VAL(rc2);
8448	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8449	Log2(("#GP rc=%Rrc\n", rc));
8450	break;
8451	}
8452	}
8453	}
8454	else
8455	rc = VERR_EM_INTERPRETER;
8456
8457	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8458	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8459	("#GP Unexpected rc=%Rrc\n", rc));
8460	return rc;
8461	}
8462
8463
8464	/**
8465	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8466	* the exception reported in the VMX transient structure back into the VM.
8467	*
8468	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8469	* VMX transient structure to be up-to-date.
8470	*/
8471	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8472	{
8473	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8474
8475	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8476	hmR0VmxCheckExitDueToEventDelivery(). */
8477	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8478	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8479	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8480	AssertRCReturn(rc, rc);
8481	return rc;
8482	}
8483
8484
8485	/**
8486	* VM-exit exception handler for #PF (Page-fault exception).
8487	*/
8488	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8489	{
8490	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8491
8492	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8493	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8494	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8495	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8496	AssertRCReturn(rc, rc);
8497
8498	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8499	if (pVM->hm.s.fNestedPaging)
8500	{
8501	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8502	{
8503	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8504	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8505	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8506	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8507	AssertRCReturn(rc, rc);
8508	}
8509	else
8510	{
8511	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8512	Assert(!pVCpu->hm.s.Event.fPending);
8513	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8514	}
8515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8516	return rc;
8517	}
8518	#else
8519	Assert(!pVM->hm.s.fNestedPaging);
8520	#endif
8521
8522	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8523	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8524	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8525	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8526	AssertRCReturn(rc, rc);
8527	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8528	if ( pVM->hm.s.fTRPPatchingAllowed
8529	&& pVM->hm.s.pGuestPatchMem
8530	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8531	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8532	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8533	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8534	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8535	{
8536	RTGCPHYS GCPhys;
8537	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8538	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8539	if ( rc == VINF_SUCCESS
8540	&& GCPhys == GCPhysApicBase)
8541	{
8542	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8543	AssertRCReturn(rc, rc);
8544
8545	/* Only attempt to patch the instruction once. */
8546	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8547	if (!pPatch)
8548	return VINF_EM_HM_PATCH_TPR_INSTR;
8549	}
8550	}
8551	#endif
8552
8553	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8554	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8555	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8556
8557	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8558	AssertRCReturn(rc, rc);
8559
8560	/* Forward it to the trap handler first. */
8561	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8562	(RTGCPTR)pVmxTransient->uExitQualification);
8563
8564	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8565	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8566
8567	if (rc == VINF_SUCCESS)
8568	{
8569	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8570	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8571	* memory? We don't update the whole state here... */
8572	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8573	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8574
8575	TRPMResetTrap(pVCpu);
8576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8577	return rc;
8578	}
8579	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8580	{
8581	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8582	{
8583	/* It's a guest page fault and needs to be reflected to the guest. */
8584	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8585	TRPMResetTrap(pVCpu);
8586	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8587	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8588	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8589	pVmxTransient->cbInstr, uGstErrorCode);
8590	AssertRCReturn(rc, rc);
8591	}
8592	else
8593	{
8594	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8595	Assert(!pVCpu->hm.s.Event.fPending);
8596	TRPMResetTrap(pVCpu);
8597	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8598	}
8599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8600	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8601	return rc;
8602	}
8603
8604	TRPMResetTrap(pVCpu);
8605	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8606	return rc;
8607	}
8608

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45413

Download in other formats: