HMVMXR0.cpp@ 45379

Last change on this file since 45379 was 45379, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits, optimizations, be more aggressive with saving state for hot exit paths. Do less on every exit, moved pending event delivery to only the required exits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 361.4 KB

Line
1	/* $Id: HMVMXR0.cpp 45379 2013-04-05 14:46:04Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
76	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(19)
77	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(21)
79	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
80	\| VMX_UPDATED_GUEST_RIP \
81	\| VMX_UPDATED_GUEST_RSP \
82	\| VMX_UPDATED_GUEST_RFLAGS \
83	\| VMX_UPDATED_GUEST_CR0 \
84	\| VMX_UPDATED_GUEST_CR3 \
85	\| VMX_UPDATED_GUEST_CR4 \
86	\| VMX_UPDATED_GUEST_GDTR \
87	\| VMX_UPDATED_GUEST_IDTR \
88	\| VMX_UPDATED_GUEST_LDTR \
89	\| VMX_UPDATED_GUEST_TR \
90	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
91	\| VMX_UPDATED_GUEST_DEBUG \
92	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
96	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
97	\| VMX_UPDATED_GUEST_INTR_STATE \
98	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
99	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
100	\| VMX_UPDATED_GUEST_APIC_STATE)
101
102	/**
103	* Flags to skip redundant reads of some common VMCS fields.
104	*/
105	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
106	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
107	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
108	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
109	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
110	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
111
112	/*
113	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
114	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
115	* we have Nested Paging support.
116	*/
117	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/* Maximum VM-instruction error number. */
126	#define VMX_INSTR_ERROR_MAX 28
127
128	/*******************************************************************************
129	* Structures and Typedefs *
130	*******************************************************************************/
131	/**
132	* A state structure for holding miscellaneous information across
133	* VMX non-root operation and restored after the transition.
134	*/
135	typedef struct VMXTRANSIENT
136	{
137	/** The host's rflags/eflags. */
138	RTCCUINTREG uEFlags;
139	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
140	uint64_t u64LStarMsr;
141	/** The guest's TPR value used for TPR shadowing. */
142	uint8_t u8GuestTpr;
143
144	/** The basic VM-exit reason. */
145	uint16_t uExitReason;
146	/** The VM-exit exit qualification. */
147	RTGCUINTPTR uExitQualification;
148	/** The VM-exit interruption error code. */
149	uint32_t uExitIntrErrorCode;
150
151	/** The VM-exit interruption-information field. */
152	uint32_t uExitIntrInfo;
153	/** Whether the VM-entry failed or not. */
154	bool fVMEntryFailed;
155	/** The VM-exit instruction-length field. */
156	uint32_t cbInstr;
157
158	/** The VM-entry interruption-information field. */
159	uint32_t uEntryIntrInfo;
160	/** The VM-entry exception error code field. */
161	uint32_t uEntryXcptErrorCode;
162	/** The VM-entry instruction length field. */
163	uint32_t cbEntryInstr;
164
165	/** IDT-vectoring information field. */
166	uint32_t uIdtVectoringInfo;
167	/** IDT-vectoring error code. */
168	uint32_t uIdtVectoringErrorCode;
169
170	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
171	uint32_t fVmcsFieldsRead;
172	/** Whether TSC-offsetting should be setup before VM-entry. */
173	bool fUpdateTscOffsetting;
174	/** Whether the VM-exit was caused by a page-fault during delivery of a
175	* contributary exception or a page-fault. */
176	bool fVectoringPF;
177	} VMXTRANSIENT, *PVMXTRANSIENT;
178
179	/**
180	* MSR-bitmap read permissions.
181	*/
182	typedef enum VMXMSREXITREAD
183	{
184	/** Reading this MSR causes a VM-exit. */
185	VMXMSREXIT_INTERCEPT_READ = 0xb,
186	/** Reading this MSR does not cause a VM-exit. */
187	VMXMSREXIT_PASSTHRU_READ
188	} VMXMSREXITREAD;
189
190	/**
191	* MSR-bitmap write permissions.
192	*/
193	typedef enum VMXMSREXITWRITE
194	{
195	/** Writing to this MSR causes a VM-exit. */
196	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
197	/** Writing to this MSR does not cause a VM-exit. */
198	VMXMSREXIT_PASSTHRU_WRITE
199	} VMXMSREXITWRITE;
200
201	/*******************************************************************************
202	* Internal Functions *
203	*******************************************************************************/
204	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
205	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
206	uint32_t cbInstr, uint32_t u32ErrCode);
207
208	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259
260	/*******************************************************************************
261	* Global Variables *
262	*******************************************************************************/
263	/** @todo Move this to hm_vmx.h. */
264	/**
265	* VM-exit handler.
266	*
267	* @returns VBox status code.
268	* @param pVM Pointer to the VM.
269	* @param pVCpu Pointer to the VMCPU.
270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
271	* out-of-sync. Make sure to update the required
272	* fields before using them.
273	* @param pVmxTransient Pointer to the VMX-transient structure.
274	*/
275	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
278
279	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
280	{
281	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
282	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
283	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
284	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
285	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
286	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
287	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
288	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
289	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
290	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
291	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
292	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
293	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
294	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
295	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
296	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
297	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
298	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
299	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
300	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
301	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
302	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
303	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
304	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
305	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
306	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
307	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
308	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
309	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
310	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
311	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
312	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
313	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
314	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
315	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
316	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
317	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
318	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
319	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
320	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
321	/* 40 UNDEFINED */ hmR0VmxExitPause,
322	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
323	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
324	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
325	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
326	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
327	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
328	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
329	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
330	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
331	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
332	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
333	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
334	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
335	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
336	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
337	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
338	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
339	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
340	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
341	};
342
343	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
344	{
345	/* 0 */ "(Not Used)",
346	/* 1 */ "VMCALL executed in VMX root operation.",
347	/* 2 */ "VMCLEAR with invalid physical address.",
348	/* 3 */ "VMCLEAR with VMXON pointer.",
349	/* 4 */ "VMLAUNCH with non-clear VMCS.",
350	/* 5 */ "VMRESUME with non-launched VMCS.",
351	/* 6 */ "VMRESUME after VMXOFF",
352	/* 7 */ "VM entry with invalid control fields.",
353	/* 8 */ "VM entry with invalid host state fields.",
354	/* 9 */ "VMPTRLD with invalid physical address.",
355	/* 10 */ "VMPTRLD with VMXON pointer.",
356	/* 11 */ "VMPTRLD with incorrect revision identifier.",
357	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
358	/* 13 */ "VMWRITE to read-only VMCS component.",
359	/* 14 */ "(Not Used)",
360	/* 15 */ "VMXON executed in VMX root operation.",
361	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
362	/* 17 */ "VM entry with non-launched executing VMCS.",
363	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
364	/* 19 */ "VMCALL with non-clear VMCS.",
365	/* 20 */ "VMCALL with invalid VM-exit control fields.",
366	/* 21 */ "(Not Used)",
367	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
368	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
369	/* 24 */ "VMCALL with invalid SMM-monitor features.",
370	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
371	/* 26 */ "VM entry with events blocked by MOV SS.",
372	/* 27 */ "(Not Used)",
373	/* 28 */ "Invalid operand to INVEPT/INVVPID."
374	};
375
376
377	/**
378	* Disables longjmps to ring-3.
379	* @param pVCpu Pointer to the VMCPU.
380	*/
381	DECLINLINE(void) hmR0VmxCallRing3Disable(PVMCPU pVCpu)
382	{
383	while (VMMRZCallRing3IsEnabled(pVCpu))
384	VMMRZCallRing3Disable(pVCpu);
385	Assert(VMMR0IsLogFlushDisabled(pVCpu));
386	}
387
388
389	/**
390	* Enables longjmps to ring-3.
391	* @param pVCpu Pointer to the VMCPU.
392	*/
393	DECLINLINE(void) hmR0VmxCallRing3Enable(PVMCPU pVCpu)
394	{
395	while (!VMMRZCallRing3IsEnabled(pVCpu))
396	VMMRZCallRing3Enable(pVCpu);
397	Assert(!VMMR0IsLogFlushDisabled(pVCpu));
398	}
399
400
401	/**
402	* Updates the VM's last error record. If there was a VMX instruction error,
403	* reads the error data from the VMCS and updates VCPU's last error record as
404	* well.
405	*
406	* @param pVM Pointer to the VM.
407	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
408	* VERR_VMX_GENERIC).
409	* @param rc The error code.
410	*/
411	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
412	{
413	AssertPtr(pVM);
414	if (rc == VERR_VMX_GENERIC)
415	{
416	AssertPtrReturnVoid(pVCpu);
417	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
418	}
419	pVM->hm.s.lLastError = rc;
420	}
421
422
423	/**
424	* Reads the VM-entry interruption-information field from the VMCS into the VMX
425	* transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVmxTransient Pointer to the VMX transient structure.
446	*
447	* @remarks No-long-jump zone!!!
448	*/
449	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
450	{
451	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
452	AssertRCReturn(rc, rc);
453	return VINF_SUCCESS;
454	}
455
456
457	/**
458	* Reads the VM-entry exception error code field from the VMCS into
459	* the VMX transient structure.
460	*
461	* @returns VBox status code.
462	* @param pVmxTransient Pointer to the VMX transient structure.
463	*
464	* @remarks No-long-jump zone!!!
465	*/
466	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
467	{
468	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
469	AssertRCReturn(rc, rc);
470	return VINF_SUCCESS;
471	}
472
473
474	/**
475	* Reads the VM-exit interruption-information field from the VMCS into the VMX
476	* transient structure.
477	*
478	* @returns VBox status code.
479	* @param pVmxTransient Pointer to the VMX transient structure.
480	*/
481	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
482	{
483	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
484	{
485	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
486	AssertRCReturn(rc, rc);
487	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
488	}
489	return VINF_SUCCESS;
490	}
491
492
493	/**
494	* Reads the VM-exit interruption error code from the VMCS into the VMX
495	* transient structure.
496	*
497	* @returns VBox status code.
498	* @param pVmxTransient Pointer to the VMX transient structure.
499	*/
500	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
501	{
502	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
503	{
504	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
505	AssertRCReturn(rc, rc);
506	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
507	}
508	return VINF_SUCCESS;
509	}
510
511
512	/**
513	* Reads the VM-exit instruction length field from the VMCS into the VMX
514	* transient structure.
515	*
516	* @returns VBox status code.
517	* @param pVmxTransient Pointer to the VMX transient structure.
518	*/
519	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
520	{
521	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
522	{
523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
524	AssertRCReturn(rc, rc);
525	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
526	}
527	return VINF_SUCCESS;
528	}
529
530
531	/**
532	* Reads the exit qualification from the VMCS into the VMX transient structure.
533	*
534	* @returns VBox status code.
535	* @param pVmxTransient Pointer to the VMX transient structure.
536	*/
537	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
538	{
539	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
540	{
541	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
542	AssertRCReturn(rc, rc);
543	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
544	}
545	return VINF_SUCCESS;
546	}
547
548
549	/**
550	* Reads the IDT-vectoring information field from the VMCS into the VMX
551	* transient structure.
552	*
553	* @returns VBox status code.
554	* @param pVmxTransient Pointer to the VMX transient structure.
555	*
556	* @remarks No-long-jump zone!!!
557	*/
558	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
559	{
560	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
561	{
562	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
563	AssertRCReturn(rc, rc);
564	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
565	}
566	return VINF_SUCCESS;
567	}
568
569
570	/**
571	* Reads the IDT-vectoring error code from the VMCS into the VMX
572	* transient structure.
573	*
574	* @returns VBox status code.
575	* @param pVmxTransient Pointer to the VMX transient structure.
576	*/
577	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
578	{
579	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
580	{
581	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
582	AssertRCReturn(rc, rc);
583	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
584	}
585	return VINF_SUCCESS;
586	}
587
588
589	/**
590	* Enters VMX root mode operation on the current CPU.
591	*
592	* @returns VBox status code.
593	* @param pVM Pointer to the VM (optional, can be NULL, after
594	* a resume).
595	* @param HCPhysCpuPage Physical address of the VMXON region.
596	* @param pvCpuPage Pointer to the VMXON region.
597	*/
598	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
599	{
600	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
601	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
602
603	if (pVM)
604	{
605	/* Write the VMCS revision dword to the VMXON region. */
606	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
607	}
608
609	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
610	RTCCUINTREG fFlags = ASMIntDisableFlags();
611
612	/* Enable the VMX bit in CR4. */
613	RTCCUINTREG uCr4 = ASMGetCR4();
614	if (!(uCr4 & X86_CR4_VMXE))
615	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
616
617	/* Enter VMXON root mode. */
618	int rc = VMXEnable(HCPhysCpuPage);
619
620	/* Restore interrupts. */
621	ASMSetFlags(fFlags);
622	return rc;
623	}
624
625
626	/**
627	* Exits VMX root mode operation on the current CPU.
628	*
629	* @returns VBox status code.
630	*/
631	static int hmR0VmxLeaveRootMode(void)
632	{
633	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
634	RTCCUINTREG fFlags = ASMIntDisableFlags();
635	int rc = VINF_SUCCESS;
636
637	/* If we're for some reason not in VMX root mode, then don't leave it. */
638	if (ASMGetCR4() & X86_CR4_VMXE)
639	{
640	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
641	VMXDisable();
642	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
643	}
644	else
645	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
646
647	/* Restore interrupts. */
648	ASMSetFlags(fFlags);
649	return rc;
650	}
651
652
653	/**
654	* Allocates and maps one physically contiguous page. The allocated page is
655	* zero'd out. (Used by various VT-x structures).
656	*
657	* @returns IPRT status code.
658	* @param pMemObj Pointer to the ring-0 memory object.
659	* @param ppVirt Where to store the virtual address of the
660	* allocation.
661	* @param pPhys Where to store the physical address of the
662	* allocation.
663	*/
664	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
665	{
666	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
667	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
668	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
669
670	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
671	if (RT_FAILURE(rc))
672	return rc;
673	ppVirt = RTR0MemObjAddress(pMemObj);
674	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
675	ASMMemZero32(*ppVirt, PAGE_SIZE);
676	return VINF_SUCCESS;
677	}
678
679
680	/**
681	* Frees and unmaps an allocated physical page.
682	*
683	* @param pMemObj Pointer to the ring-0 memory object.
684	* @param ppVirt Where to re-initialize the virtual address of
685	* allocation as 0.
686	* @param pHCPhys Where to re-initialize the physical address of the
687	* allocation as 0.
688	*/
689	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
690	{
691	AssertPtr(pMemObj);
692	AssertPtr(ppVirt);
693	AssertPtr(pHCPhys);
694	if (*pMemObj != NIL_RTR0MEMOBJ)
695	{
696	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
697	AssertRC(rc);
698	*pMemObj = NIL_RTR0MEMOBJ;
699	*ppVirt = 0;
700	*pHCPhys = 0;
701	}
702	}
703
704
705	/**
706	* Worker function to free VT-x related structures.
707	*
708	* @returns IPRT status code.
709	* @param pVM Pointer to the VM.
710	*/
711	static void hmR0VmxStructsFree(PVM pVM)
712	{
713	for (VMCPUID i = 0; i < pVM->cCpus; i++)
714	{
715	PVMCPU pVCpu = &pVM->aCpus[i];
716	AssertPtr(pVCpu);
717
718	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
720
721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
722	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
723
724	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
726	}
727
728	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
729	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
730	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
731	#endif
732	}
733
734
735	/**
736	* Worker function to allocate VT-x related VM structures.
737	*
738	* @returns IPRT status code.
739	* @param pVM Pointer to the VM.
740	*/
741	static int hmR0VmxStructsAlloc(PVM pVM)
742	{
743	/*
744	* Initialize members up-front so we can cleanup properly on allocation failure.
745	*/
746	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
747	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
748	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
749	pVM->hm.s.vmx.HCPhys##a_Name = 0;
750
751	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
752	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
753	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
754	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
755
756	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
757	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
758	#endif
759	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
760
761	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
762	for (VMCPUID i = 0; i < pVM->cCpus; i++)
763	{
764	PVMCPU pVCpu = &pVM->aCpus[i];
765	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
766	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
767	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
770	}
771	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
772	#undef VMXLOCAL_INIT_VM_MEMOBJ
773
774	/*
775	* Allocate all the VT-x structures.
776	*/
777	int rc = VINF_SUCCESS;
778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
780	if (RT_FAILURE(rc))
781	goto cleanup;
782	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
783	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
784	#endif
785
786	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
787	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
788	{
789	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
790	&pVM->hm.s.vmx.HCPhysApicAccess);
791	if (RT_FAILURE(rc))
792	goto cleanup;
793	}
794
795	/*
796	* Initialize per-VCPU VT-x structures.
797	*/
798	for (VMCPUID i =0; i < pVM->cCpus; i++)
799	{
800	PVMCPU pVCpu = &pVM->aCpus[i];
801	AssertPtr(pVCpu);
802
803	/* Allocate the VM control structure (VMCS). */
804	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
805	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
806	if (RT_FAILURE(rc))
807	goto cleanup;
808
809	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
810	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
811	{
812	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
813	&pVCpu->hm.s.vmx.HCPhysVirtApic);
814	if (RT_FAILURE(rc))
815	goto cleanup;
816	}
817
818	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
820	{
821	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
822	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
826	}
827
828	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
830	if (RT_FAILURE(rc))
831	goto cleanup;
832
833	/* Allocate the VM-exit MSR-load page for the host MSRs. */
834	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	return VINF_SUCCESS;
840
841	cleanup:
842	hmR0VmxStructsFree(pVM);
843	return rc;
844	}
845
846
847	/**
848	* Does global VT-x initialization (called during module initialization).
849	*
850	* @returns VBox status code.
851	*/
852	VMMR0DECL(int) VMXR0GlobalInit(void)
853	{
854	/* Setup the main VM exit handlers. */
855	/** @todo I'm told lookup table for function pointers is a bad idea in terms
856	* of performance. Make these static for now and not do change anything
857	* at runtime so we can easily switch to a switch-case approach later. */
858	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
859	#ifdef DEBUG
860	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
861	Assert(s_apfnVMExitHandlers[i]);
862	#endif
863	return VINF_SUCCESS;
864	}
865
866
867	/**
868	* Does global VT-x termination (called during module termination).
869	*/
870	VMMR0DECL(void) VMXR0GlobalTerm()
871	{
872	/* Nothing to do currently. */
873	}
874
875
876	/**
877	* Sets up and activates VT-x on the current CPU.
878	*
879	* @returns VBox status code.
880	* @param pCpu Pointer to the global CPU info struct.
881	* @param pVM Pointer to the VM (can be NULL after a host resume
882	* operation).
883	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
884	* fEnabledByHost is true).
885	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
886	* @a fEnabledByHost is true).
887	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
888	* enable VT-x/AMD-V on the host.
889	*/
890	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
891	{
892	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
893	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
894
895	if (!fEnabledByHost)
896	{
897	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
898	if (RT_FAILURE(rc))
899	return rc;
900	}
901
902	/*
903	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
904	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
905	* each time while reusing a VPID after hitting the MaxASID limit once.
906	*/
907	if ( pVM
908	&& pVM->hm.s.vmx.fVpid
909	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
910	{
911	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
912	pCpu->fFlushAsidBeforeUse = false;
913	}
914	else
915	pCpu->fFlushAsidBeforeUse = true;
916
917	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
918	++pCpu->cTlbFlushes;
919
920	return VINF_SUCCESS;
921	}
922
923
924	/**
925	* Deactivates VT-x on the current CPU.
926	*
927	* @returns VBox status code.
928	* @param pCpu Pointer to the global CPU info struct.
929	* @param pvCpuPage Pointer to the VMXON region.
930	* @param HCPhysCpuPage Physical address of the VMXON region.
931	*/
932	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
933	{
934	NOREF(pCpu);
935	NOREF(pvCpuPage);
936	NOREF(HCPhysCpuPage);
937
938	hmR0VmxLeaveRootMode();
939	return VINF_SUCCESS;
940	}
941
942
943	/**
944	* Sets the permission bits for the specified MSR in the MSR bitmap.
945	*
946	* @param pVCpu Pointer to the VMCPU.
947	* @param ulMSR The MSR value.
948	* @param enmRead Whether reading this MSR causes a VM-exit.
949	* @param enmWrite Whether writing this MSR causes a VM-exit.
950	*/
951	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
952	{
953	unsigned ulBit;
954	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
955
956	/*
957	* Layout:
958	* 0x000 - 0x3ff - Low MSR read bits
959	* 0x400 - 0x7ff - High MSR read bits
960	* 0x800 - 0xbff - Low MSR write bits
961	* 0xc00 - 0xfff - High MSR write bits
962	*/
963	if (ulMsr <= 0x00001FFF)
964	{
965	/* Pentium-compatible MSRs */
966	ulBit = ulMsr;
967	}
968	else if ( ulMsr >= 0xC0000000
969	&& ulMsr <= 0xC0001FFF)
970	{
971	/* AMD Sixth Generation x86 Processor MSRs */
972	ulBit = (ulMsr - 0xC0000000);
973	pbMsrBitmap += 0x400;
974	}
975	else
976	{
977	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
978	return;
979	}
980
981	Assert(ulBit <= 0x1fff);
982	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
983	ASMBitSet(pbMsrBitmap, ulBit);
984	else
985	ASMBitClear(pbMsrBitmap, ulBit);
986
987	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
988	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
989	else
990	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
991	}
992
993
994	/**
995	* Flushes the TLB using EPT.
996	*
997	* @returns VBox status code.
998	* @param pVM Pointer to the VM.
999	* @param pVCpu Pointer to the VMCPU.
1000	* @param enmFlush Type of flush.
1001	*/
1002	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1003	{
1004	AssertPtr(pVM);
1005	Assert(pVM->hm.s.fNestedPaging);
1006
1007	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1008
1009	uint64_t descriptor[2];
1010	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
1011	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1012
1013	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1014	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1015	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1016	}
1017
1018
1019	/**
1020	* Flushes the TLB using VPID.
1021	*
1022	* @returns VBox status code.
1023	* @param pVM Pointer to the VM.
1024	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1025	* enmFlush).
1026	* @param enmFlush Type of flush.
1027	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1028	* on @a enmFlush).
1029	*/
1030	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.vmx.fVpid);
1034
1035	uint64_t descriptor[2];
1036	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1037	{
1038	descriptor[0] = 0;
1039	descriptor[1] = 0;
1040	}
1041	else
1042	{
1043	AssertPtr(pVCpu);
1044	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1045	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1046	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1047	descriptor[1] = GCPtr;
1048	}
1049
1050	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1051	AssertMsg(rc == VINF_SUCCESS,
1052	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1053	if ( RT_SUCCESS(rc)
1054	&& pVCpu)
1055	{
1056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1057	}
1058	}
1059
1060
1061	/**
1062	* Invalidates a guest page by guest virtual address. Only relevant for
1063	* EPT/VPID, otherwise there is nothing really to invalidate.
1064	*
1065	* @returns VBox status code.
1066	* @param pVM Pointer to the VM.
1067	* @param pVCpu Pointer to the VMCPU.
1068	* @param GCVirt Guest virtual address of the page to invalidate.
1069	*/
1070	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1071	{
1072	AssertPtr(pVM);
1073	AssertPtr(pVCpu);
1074	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1075
1076	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1077	if (!fFlushPending)
1078	{
1079	/*
1080	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1081	* See @bugref{6043} and @bugref{6177}.
1082	*
1083	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1084	* function maybe called in a loop with individual addresses.
1085	*/
1086	if (pVM->hm.s.vmx.fVpid)
1087	{
1088	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1089	{
1090	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1091	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1092	}
1093	else
1094	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1095	}
1096	else if (pVM->hm.s.fNestedPaging)
1097	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1098	}
1099
1100	return VINF_SUCCESS;
1101	}
1102
1103
1104	/**
1105	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1106	* otherwise there is nothing really to invalidate.
1107	*
1108	* @returns VBox status code.
1109	* @param pVM Pointer to the VM.
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param GCPhys Guest physical address of the page to invalidate.
1112	*/
1113	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1114	{
1115	LogFlowFunc(("%RGp\n", GCPhys));
1116
1117	/*
1118	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1119	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1120	* This function might be called in a loop.
1121	*/
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1130	* case where neither EPT nor VPID is supported by the CPU.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	*
1135	* @remarks Called with interrupts disabled.
1136	*/
1137	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1138	{
1139	NOREF(pVM);
1140	AssertPtr(pVCpu);
1141	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1143
1144	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1145	AssertPtr(pCpu);
1146
1147	pVCpu->hm.s.TlbShootdown.cPages = 0;
1148	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1149	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1150	pVCpu->hm.s.fForceTLBFlush = false;
1151	return;
1152	}
1153
1154
1155	/**
1156	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1157	*
1158	* @param pVM Pointer to the VM.
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @remarks All references to "ASID" in this function pertains to "VPID" in
1161	* Intel's nomenclature. The reason is, to avoid confusion in compare
1162	* statements since the host-CPU copies are named "ASID".
1163	*
1164	* @remarks Called with interrupts disabled.
1165	* @todo Statistics.
1166	*/
1167	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1168	{
1169	AssertPtr(pVM);
1170	AssertPtr(pVCpu);
1171	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1172	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1173	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1174
1175	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1176	AssertPtr(pCpu);
1177
1178	/*
1179	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1180	* This can happen both for start & resume due to long jumps back to ring-3.
1181	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1182	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1183	*/
1184	bool fNewASID = false;
1185	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1186	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1187	{
1188	pVCpu->hm.s.fForceTLBFlush = true;
1189	fNewASID = true;
1190	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1191	}
1192
1193	/*
1194	* Check for explicit TLB shootdowns.
1195	*/
1196	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1197	{
1198	pVCpu->hm.s.fForceTLBFlush = true;
1199	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1200	}
1201
1202	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1203	if (pVCpu->hm.s.fForceTLBFlush)
1204	{
1205	if (fNewASID)
1206	{
1207	++pCpu->uCurrentAsid;
1208	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1209	{
1210	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1211	pCpu->cTlbFlushes++;
1212	pCpu->fFlushAsidBeforeUse = true;
1213	}
1214
1215	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1216	if (pCpu->fFlushAsidBeforeUse)
1217	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1218	}
1219	else
1220	{
1221	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1222	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1223	else
1224	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1225	}
1226
1227	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1228	pVCpu->hm.s.fForceTLBFlush = false;
1229	}
1230	else
1231	{
1232	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1233	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1234	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1235	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1236
1237	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1238	* not be executed. See hmQueueInvlPage() where it is commented
1239	* out. Support individual entry flushing someday. */
1240	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1241	{
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1243
1244	/*
1245	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1246	* as supported by the CPU.
1247	*/
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1249	{
1250	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1251	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1252	}
1253	else
1254	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1255	}
1256	else
1257	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1258	}
1259	pVCpu->hm.s.TlbShootdown.cPages = 0;
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1263	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1264	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1265	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1266	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1267	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1268
1269	/* Update VMCS with the VPID. */
1270	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1271	AssertRC(rc);
1272	}
1273
1274
1275	/**
1276	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1277	*
1278	* @returns VBox status code.
1279	* @param pVM Pointer to the VM.
1280	* @param pVCpu Pointer to the VMCPU.
1281	*
1282	* @remarks Called with interrupts disabled.
1283	* @todo Statistics.
1284	*/
1285	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1286	{
1287	AssertPtr(pVM);
1288	AssertPtr(pVCpu);
1289	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1290	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1291
1292	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1293	AssertPtr(pCpu);
1294
1295	/*
1296	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1297	* This can happen both for start & resume due to long jumps back to ring-3.
1298	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1299	*/
1300	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1301	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1302	{
1303	pVCpu->hm.s.fForceTLBFlush = true;
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1305	}
1306
1307	/* Check for explicit TLB shootdown flushes. */
1308	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1309	{
1310	pVCpu->hm.s.fForceTLBFlush = true;
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1312	}
1313
1314	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1315	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1316
1317	if (pVCpu->hm.s.fForceTLBFlush)
1318	{
1319	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1320	pVCpu->hm.s.fForceTLBFlush = false;
1321	}
1322	else
1323	{
1324	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1325	* not be executed. See hmQueueInvlPage() where it is commented
1326	* out. Support individual entry flushing someday. */
1327	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1328	{
1329	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	}
1333	else
1334	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1335	}
1336
1337	pVCpu->hm.s.TlbShootdown.cPages= 0;
1338	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1339	}
1340
1341
1342	/**
1343	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1344	*
1345	* @returns VBox status code.
1346	* @param pVM Pointer to the VM.
1347	* @param pVCpu Pointer to the VMCPU.
1348	*
1349	* @remarks Called with interrupts disabled.
1350	* @todo Statistics.
1351	*/
1352	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1353	{
1354	AssertPtr(pVM);
1355	AssertPtr(pVCpu);
1356	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1357	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1358
1359	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1360
1361	/*
1362	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1363	* This can happen both for start & resume due to long jumps back to ring-3.
1364	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1365	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1366	*/
1367	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1368	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1369	{
1370	pVCpu->hm.s.fForceTLBFlush = true;
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1372	}
1373
1374	/* Check for explicit TLB shootdown flushes. */
1375	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1376	{
1377	/*
1378	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1379	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1380	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1381	*/
1382	pVCpu->hm.s.fForceTLBFlush = true;
1383	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1384	}
1385
1386	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1387	if (pVCpu->hm.s.fForceTLBFlush)
1388	{
1389	++pCpu->uCurrentAsid;
1390	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1391	{
1392	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1393	pCpu->fFlushAsidBeforeUse = true;
1394	pCpu->cTlbFlushes++;
1395	}
1396
1397	pVCpu->hm.s.fForceTLBFlush = false;
1398	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1399	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1400	if (pCpu->fFlushAsidBeforeUse)
1401	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1402	}
1403	else
1404	{
1405	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1406	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1407	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1408	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1409
1410	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1411	* not be executed. See hmQueueInvlPage() where it is commented
1412	* out. Support individual entry flushing someday. */
1413	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1414	{
1415	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1416	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1417	{
1418	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1419	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1420	}
1421	else
1422	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1423	}
1424	else
1425	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1426	}
1427
1428	pVCpu->hm.s.TlbShootdown.cPages = 0;
1429	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1430
1431	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1432	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1433	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1434	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1435	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1436	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1437
1438	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1439	AssertRC(rc);
1440	}
1441
1442
1443	/**
1444	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1445	* TLB entries from the host TLB before VM-entry.
1446	*
1447	* @returns VBox status code.
1448	* @param pVM Pointer to the VM.
1449	*/
1450	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1451	{
1452	/*
1453	* Determine optimal flush type for nested paging.
1454	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1455	* guest execution (see hmR3InitFinalizeR0()).
1456	*/
1457	if (pVM->hm.s.fNestedPaging)
1458	{
1459	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1460	{
1461	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1462	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1463	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1464	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1465	else
1466	{
1467	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1468	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1469	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1470	}
1471
1472	/* Make sure the write-back cacheable memory type for EPT is supported. */
1473	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1474	{
1475	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1476	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1477	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1478	}
1479	}
1480	else
1481	{
1482	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1483	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1484	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1485	}
1486	}
1487
1488	/*
1489	* Determine optimal flush type for VPID.
1490	*/
1491	if (pVM->hm.s.vmx.fVpid)
1492	{
1493	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1494	{
1495	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1496	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1497	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1499	else
1500	{
1501	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1502	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1503	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1504	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1505	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1506	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1507	pVM->hm.s.vmx.fVpid = false;
1508	}
1509	}
1510	else
1511	{
1512	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1513	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1515	pVM->hm.s.vmx.fVpid = false;
1516	}
1517	}
1518
1519	/*
1520	* Setup the handler for flushing tagged-TLBs.
1521	*/
1522	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1523	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1524	else if (pVM->hm.s.fNestedPaging)
1525	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1526	else if (pVM->hm.s.vmx.fVpid)
1527	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1528	else
1529	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1530	return VINF_SUCCESS;
1531	}
1532
1533
1534	/**
1535	* Sets up pin-based VM-execution controls in the VMCS.
1536	*
1537	* @returns VBox status code.
1538	* @param pVM Pointer to the VM.
1539	* @param pVCpu Pointer to the VMCPU.
1540	*/
1541	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1542	{
1543	AssertPtr(pVM);
1544	AssertPtr(pVCpu);
1545
1546	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1547	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1548
1549	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1550	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1551	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1552
1553	/* Enable the VMX preemption timer. */
1554	if (pVM->hm.s.vmx.fUsePreemptTimer)
1555	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1556
1557	if ((val & zap) != val)
1558	{
1559	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1560	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1562	}
1563
1564	val &= zap;
1565	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1566	AssertRCReturn(rc, rc);
1567
1568	/* Update VCPU with the currently set pin-based VM-execution controls. */
1569	pVCpu->hm.s.vmx.u32PinCtls = val;
1570	return rc;
1571	}
1572
1573
1574	/**
1575	* Sets up processor-based VM-execution controls in the VMCS.
1576	*
1577	* @returns VBox status code.
1578	* @param pVM Pointer to the VM.
1579	* @param pVMCPU Pointer to the VMCPU.
1580	*/
1581	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1582	{
1583	AssertPtr(pVM);
1584	AssertPtr(pVCpu);
1585
1586	int rc = VERR_INTERNAL_ERROR_5;
1587	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1588	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1589
1590	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1591	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1592	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1593	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1594	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1595	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1596	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1597
1598	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1599	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1600	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1601	{
1602	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1603	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1604	}
1605
1606	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1607	if (!pVM->hm.s.fNestedPaging)
1608	{
1609	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1613	}
1614
1615	/* Use TPR shadowing if supported by the CPU. */
1616	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1617	{
1618	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1619	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1620	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1622
1623	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1624	/* CR8 writes causes a VM-exit based on TPR threshold. */
1625	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1626	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1627	}
1628	else
1629	{
1630	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1632	}
1633
1634	/* Use MSR-bitmaps if supported by the CPU. */
1635	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1636	{
1637	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1638
1639	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1640	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1641	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1642	AssertRCReturn(rc, rc);
1643
1644	/*
1645	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1646	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1647	*/
1648	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1649	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1650	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1651	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1652	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1653	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1654	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1655	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1656	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1657	}
1658
1659	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1660	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1662
1663	if ((val & zap) != val)
1664	{
1665	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1666	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1667	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1668	}
1669
1670	val &= zap;
1671	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1672	AssertRCReturn(rc, rc);
1673
1674	/* Update VCPU with the currently set processor-based VM-execution controls. */
1675	pVCpu->hm.s.vmx.u32ProcCtls = val;
1676
1677	/*
1678	* Secondary processor-based VM-execution controls.
1679	*/
1680	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681	{
1682	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1683	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1684
1685	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1686
1687	if (pVM->hm.s.fNestedPaging)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1689	else
1690	{
1691	/*
1692	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1693	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1694	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1695	*/
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1698	}
1699
1700	if (pVM->hm.s.vmx.fVpid)
1701	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1702
1703	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1704	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1705
1706	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1707	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1708	* done dynamically. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1710	{
1711	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1712	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1713	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1714	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1715	AssertRCReturn(rc, rc);
1716	}
1717
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1719	{
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1722	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1723	}
1724
1725	if ((val & zap) != val)
1726	{
1727	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1728	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1729	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1730	}
1731
1732	val &= zap;
1733	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1734	AssertRCReturn(rc, rc);
1735
1736	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1737	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1738	}
1739
1740	return VINF_SUCCESS;
1741	}
1742
1743
1744	/**
1745	* Sets up miscellaneous (everything other than Pin & Processor-based
1746	* VM-execution) control fields in the VMCS.
1747	*
1748	* @returns VBox status code.
1749	* @param pVM Pointer to the VM.
1750	* @param pVCpu Pointer to the VMCPU.
1751	*/
1752	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1753	{
1754	AssertPtr(pVM);
1755	AssertPtr(pVCpu);
1756
1757	int rc = VERR_GENERAL_FAILURE;
1758
1759	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1760	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1761
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1763
1764	/*
1765	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1766	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1767	* We thus use the exception bitmap to control it rather than use both.
1768	*/
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1770	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1771
1772	/** @todo Explore possibility of using IO-bitmaps. */
1773	/* All IO & IOIO instructions cause VM-exits. */
1774	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1775	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1776
1777	/* Setup MSR autoloading/autostoring. */
1778	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1779	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1781	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1782	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1783	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1784
1785	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1786	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1787	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1788	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1789
1790	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1791	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1792
1793	/* Setup debug controls */
1794	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1795	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1796	AssertRCReturn(rc, rc);
1797	return rc;
1798	}
1799
1800
1801	/**
1802	* Sets up the initial exception bitmap in the VMCS based on static conditions
1803	* (i.e. conditions that cannot ever change at runtime).
1804	*
1805	* @returns VBox status code.
1806	* @param pVM Pointer to the VM.
1807	* @param pVCpu Pointer to the VMCPU.
1808	*/
1809	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1810	{
1811	AssertPtr(pVM);
1812	AssertPtr(pVCpu);
1813
1814	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1815
1816	uint32_t u32XcptBitmap = 0;
1817
1818	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1819	if (!pVM->hm.s.fNestedPaging)
1820	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1821
1822	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1823	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1824	AssertRCReturn(rc, rc);
1825	return rc;
1826	}
1827
1828
1829	/**
1830	* Sets up the initial guest-state mask. The guest-state mask is consulted
1831	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1832	* for the nested virtualization case (as it would cause a VM-exit).
1833	*
1834	* @param pVCpu Pointer to the VMCPU.
1835	*/
1836	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1837	{
1838	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1839	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1840	return VINF_SUCCESS;
1841	}
1842
1843
1844	/**
1845	* Does per-VM VT-x initialization.
1846	*
1847	* @returns VBox status code.
1848	* @param pVM Pointer to the VM.
1849	*/
1850	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1851	{
1852	LogFlowFunc(("pVM=%p\n", pVM));
1853
1854	int rc = hmR0VmxStructsAlloc(pVM);
1855	if (RT_FAILURE(rc))
1856	{
1857	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1858	return rc;
1859	}
1860
1861	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1862	{
1863	PVMCPU pVCpu = &pVM->aCpus[i];
1864
1865	/* Current guest paging mode. */
1866	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1867	}
1868
1869	return VINF_SUCCESS;
1870	}
1871
1872
1873	/**
1874	* Does per-VM VT-x termination.
1875	*
1876	* @returns VBox status code.
1877	* @param pVM Pointer to the VM.
1878	*/
1879	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1880	{
1881	LogFlowFunc(("pVM=%p\n", pVM));
1882
1883	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1884	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1885	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1886	#endif
1887	hmR0VmxStructsFree(pVM);
1888	return VINF_SUCCESS;
1889	}
1890
1891
1892	/**
1893	* Sets up the VM for execution under VT-x.
1894	* This function is only called once per-VM during initalization.
1895	*
1896	* @returns VBox status code.
1897	* @param pVM Pointer to the VM.
1898	*/
1899	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1900	{
1901	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1902	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1903
1904	LogFlowFunc(("pVM=%p\n", pVM));
1905
1906	/*
1907	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1908	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1909	*/
1910	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1911	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1912	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1913	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1914	{
1915	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1916	return VERR_INTERNAL_ERROR;
1917	}
1918
1919	/* Initialize these always, see hmR3InitFinalizeR0().*/
1920	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1921	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1922
1923	/* Setup the tagged-TLB flush handlers. */
1924	int rc = hmR0VmxSetupTaggedTlb(pVM);
1925	if (RT_FAILURE(rc))
1926	{
1927	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1928	return rc;
1929	}
1930
1931	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1932	{
1933	PVMCPU pVCpu = &pVM->aCpus[i];
1934	AssertPtr(pVCpu);
1935	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1936
1937	/* Set revision dword at the beginning of the VMCS structure. */
1938	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1939
1940	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1941	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1942	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1943	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1944
1945	/* Load this VMCS as the current VMCS. */
1946	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1947	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1948	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1949
1950	/* Setup the pin-based VM-execution controls. */
1951	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1952	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1953	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1954
1955	/* Setup the processor-based VM-execution controls. */
1956	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1957	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1958	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1959
1960	/* Setup the rest (miscellaneous) VM-execution controls. */
1961	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1962	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1963	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1964
1965	/* Setup the initial exception bitmap. */
1966	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1967	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1968	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1969
1970	/* Setup the initial guest-state mask. */
1971	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1972	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1973	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1974
1975	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1976	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1977	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1978	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1979	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1980	#endif
1981
1982	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1983	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1984	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1985	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1986
1987	/* Update the last error record for this VCPU. */
1988	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1989	}
1990
1991	return VINF_SUCCESS;
1992	}
1993
1994
1995	/**
1996	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1997	* the VMCS.
1998	*
1999	* @returns VBox status code.
2000	* @param pVM Pointer to the VM.
2001	* @param pVCpu Pointer to the VMCPU.
2002	*/
2003	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2004	{
2005	RTCCUINTREG uReg = ASMGetCR0();
2006	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2007	AssertRCReturn(rc, rc);
2008
2009	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2010	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2011	if (VMX_IS_64BIT_HOST_MODE())
2012	{
2013	uint64_t uReg = hmR0Get64bitCR3();
2014	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2015	}
2016	else
2017	#endif
2018	{
2019	uReg = ASMGetCR3();
2020	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2021	}
2022	AssertRCReturn(rc, rc);
2023
2024	uReg = ASMGetCR4();
2025	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2026	AssertRCReturn(rc, rc);
2027	return rc;
2028	}
2029
2030
2031	/**
2032	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2033	* the host-state area in the VMCS.
2034	*
2035	* @returns VBox status code.
2036	* @param pVM Pointer to the VM.
2037	* @param pVCpu Pointer to the VMCPU.
2038	*/
2039	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2040	{
2041	int rc = VERR_INTERNAL_ERROR_5;
2042	RTSEL uSelCS = 0;
2043	RTSEL uSelSS = 0;
2044	RTSEL uSelDS = 0;
2045	RTSEL uSelES = 0;
2046	RTSEL uSelFS = 0;
2047	RTSEL uSelGS = 0;
2048	RTSEL uSelTR = 0;
2049
2050	/*
2051	* Host Selector registers.
2052	*/
2053	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2054	if (VMX_IS_64BIT_HOST_MODE())
2055	{
2056	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2057	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2058	}
2059	else
2060	{
2061	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2062	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2063	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2064	}
2065	#else
2066	uSelCS = ASMGetCS();
2067	uSelSS = ASMGetSS();
2068	#endif
2069
2070	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2071	/** @todo Verify if we have any platform that actually run with DS or ES with
2072	* RPL != 0 in kernel space. */
2073	uSelDS = 0;
2074	uSelES = 0;
2075	uSelFS = 0;
2076	uSelGS = 0;
2077	uSelTR = ASMGetTR();
2078
2079	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2080	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2081	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2082	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2083	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2084	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2085	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2086	Assert(uSelCS != 0);
2087	Assert(uSelTR != 0);
2088
2089	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2090	#if 0
2091	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2092	Assert(uSelSS != 0);
2093	#endif
2094
2095	/* Write these host selector fields into the host-state area in the VMCS. */
2096	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2097	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2098	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2099	#if 0
2100	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2101	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2102	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2103	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2104	#endif
2105	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2106	AssertRCReturn(rc, rc);
2107
2108	/*
2109	* Host GDTR and IDTR.
2110	*/
2111	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2112	* be safe to -not- save and restore GDTR and IDTR in the assembly
2113	* code and just do it here and don't care if the limits are zapped on
2114	* VM-exit. */
2115	RTGDTR Gdtr;
2116	RT_ZERO(Gdtr);
2117	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2118	if (VMX_IS_64BIT_HOST_MODE())
2119	{
2120	X86XDTR64 Gtr64;
2121	X86XDTR64 Idtr64;
2122	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2123	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2124	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2125	Gdtr.cbGdt = Gdtr64.cb;
2126	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2127	}
2128	else
2129	#endif
2130	{
2131	RTIDTR Idtr;
2132	ASMGetGDTR(&Gdtr);
2133	ASMGetIDTR(&Idtr);
2134	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2135	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2136	}
2137	AssertRCReturn(rc, rc);
2138
2139	/*
2140	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2141	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2142	*/
2143	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2144	{
2145	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2146	return VERR_VMX_INVALID_HOST_STATE;
2147	}
2148
2149	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2150	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2151	if (VMX_IS_64BIT_HOST_MODE())
2152	{
2153	/* We need the 64-bit TR base for hybrid darwin. */
2154	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2155	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2156	}
2157	else
2158	#endif
2159	{
2160	uintptr_t uTRBase;
2161	#if HC_ARCH_BITS == 64
2162	uTRBase = X86DESC64_BASE(pDesc);
2163	#else
2164	uTRBase = X86DESC_BASE(pDesc);
2165	#endif
2166	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2167	}
2168	AssertRCReturn(rc, rc);
2169
2170	/*
2171	* Host FS base and GS base.
2172	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2173	* would take care of the bases. In 64-bit, the MSRs come into play.
2174	*/
2175	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2176	if (VMX_IS_64BIT_HOST_MODE())
2177	{
2178	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2179	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2180	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2181	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2182	AssertRCReturn(rc, rc);
2183	}
2184	#endif
2185	return rc;
2186	}
2187
2188
2189	/**
2190	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2191	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2192	* the host after every successful VM exit.
2193	*
2194	* @returns VBox status code.
2195	* @param pVM Pointer to the VM.
2196	* @param pVCpu Pointer to the VMCPU.
2197	*/
2198	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2199	{
2200	AssertPtr(pVCpu);
2201	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2202
2203	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2204	unsigned idxHostMsr = 0;
2205	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2206
2207	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2208	{
2209	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2210	pHostMsr->u32Reserved = 0;
2211	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2212	if (CPUMIsGuestInLongMode(pVCpu))
2213	{
2214	/* Must match the EFER value in our 64 bits switcher. */
2215	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2216	}
2217	else
2218	#endif
2219	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2220	pHostMsr++; idxHostMsr++;
2221	}
2222
2223	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2224	if (VMX_IS_64BIT_HOST_MODE())
2225	{
2226	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2227	pHostMsr->u32Reserved = 0;
2228	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2229	pHostMsr++; idxHostMsr++;
2230	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2231	pHostMsr->u32Reserved = 0;
2232	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2233	pHostMsr++; idxHostMsr++;
2234	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2235	pHostMsr->u32Reserved = 0;
2236	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2237	pHostMsr++; idxHostMsr++;
2238	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2239	#if 0
2240	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2241	pMsr->u32Reserved = 0;
2242	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2243	pHostMsr++; idxHostMsr++;
2244	#endif
2245	}
2246	#endif
2247
2248	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2249	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2250	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2251
2252	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2253	AssertRCReturn(rc, rc);
2254
2255	/*
2256	* Host Sysenter MSRs.
2257	*/
2258	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2259	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2260	if (VMX_IS_64BIT_HOST_MODE())
2261	{
2262	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2263	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2264	}
2265	else
2266	{
2267	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2268	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2269	}
2270	#elif HC_ARCH_BITS == 32
2271	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2272	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2273	#else
2274	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2275	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2276	#endif
2277	AssertRCReturn(rc, rc);
2278
2279	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2280	* hmR0VmxSetupExitCtls() !! */
2281	return rc;
2282	}
2283
2284
2285	/**
2286	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2287	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2288	* controls".
2289	*
2290	* @returns VBox status code.
2291	* @param pVM Pointer to the VM.
2292	* @param pVCpu Pointer to the VMCPU.
2293	* @param pCtx Pointer to the guest-CPU context.
2294	*
2295	* @remarks No-long-jump zone!!!
2296	*/
2297	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2298	{
2299	int rc = VINF_SUCCESS;
2300	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2301	{
2302	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2303	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2304
2305	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2306	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2307
2308	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2309	if (CPUMIsGuestInLongModeEx(pCtx))
2310	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2311	else
2312	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2313
2314	/*
2315	* The following should not be set (since we're not in SMM mode):
2316	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2317	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2318	*/
2319
2320	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2321	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2322	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2323
2324	if ((val & zap) != val)
2325	{
2326	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2327	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2328	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2329	}
2330
2331	val &= zap;
2332	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2333	AssertRCReturn(rc, rc);
2334
2335	/* Update VCPU with the currently set VM-exit controls. */
2336	pVCpu->hm.s.vmx.u32EntryCtls = val;
2337	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2338	}
2339	return rc;
2340	}
2341
2342
2343	/**
2344	* Sets up the VM-exit controls in the VMCS.
2345	*
2346	* @returns VBox status code.
2347	* @param pVM Pointer to the VM.
2348	* @param pVCpu Pointer to the VMCPU.
2349	*/
2350	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2351	{
2352	int rc = VINF_SUCCESS;
2353	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2354	{
2355	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2356	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2357
2358	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2359	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2360
2361	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2362	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2363	if (VMX_IS_64BIT_HOST_MODE())
2364	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2365	else
2366	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2367	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2368	if (CPUMIsGuestInLongModeEx(pCtx))
2369	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2370	else
2371	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2372	#endif
2373
2374	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2375	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2376
2377	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2378	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2379	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2380	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2381	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR,
2382	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER. */
2383
2384	if ((val & zap) != val)
2385	{
2386	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2387	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2388	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2389	}
2390
2391	val &= zap;
2392	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2393
2394	/* Update VCPU with the currently set VM-exit controls. */
2395	pVCpu->hm.s.vmx.u32ExitCtls = val;
2396	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2397	}
2398	return rc;
2399	}
2400
2401
2402	/**
2403	* Loads the guest APIC and related state.
2404	*
2405	* @returns VBox status code.
2406	* @param pVM Pointer to the VM.
2407	* @param pVCpu Pointer to the VMCPU.
2408	* @param pCtx Pointer to the guest-CPU context.
2409	*/
2410	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2411	{
2412	int rc = VINF_SUCCESS;
2413	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2414	{
2415	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2416	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2417	{
2418	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2419
2420	bool fPendingIntr = false;
2421	uint8_t u8GuestTpr = 0;
2422	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2423	AssertRCReturn(rc, rc);
2424
2425	/*
2426	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2427	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2428	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2429	* the interrupt when we VM-exit for other reasons.
2430	*/
2431	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2432	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2433	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2434	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2435
2436	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2437	AssertRCReturn(rc, rc);
2438
2439	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2440	if (pVM->hm.s.fTPRPatchingActive)
2441	{
2442	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2443	pCtx->msrLSTAR = u8GuestTpr;
2444	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2445	{
2446	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2447	if (fPendingIntr)
2448	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2449	else
2450	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451	}
2452	}
2453	}
2454
2455	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2456	}
2457	return rc;
2458	}
2459
2460
2461	/**
2462	* Loads the guest's RIP into the guest-state area in the VMCS.
2463	*
2464	* @returns VBox status code.
2465	* @param pVM Pointer to the VM.
2466	* @param pVCpu Pointer to the VMCPU.
2467	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2468	* out-of-sync. Make sure to update the required fields
2469	* before using them.
2470	*
2471	* @remarks No-long-jump zone!!!
2472	*/
2473	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2474	{
2475	int rc = VINF_SUCCESS;
2476	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2477	{
2478	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2479	AssertRCReturn(rc, rc);
2480	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2481	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2482	}
2483	return rc;
2484	}
2485
2486
2487	/**
2488	* Loads the guest's RSP into the guest-state area in the VMCS.
2489	*
2490	* @returns VBox status code.
2491	* @param pVM Pointer to the VM.
2492	* @param pVCpu Pointer to the VMCPU.
2493	* @param pCtx Pointer to the guest-CPU context.
2494	*
2495	* @remarks No-long-jump zone!!!
2496	*/
2497	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2498	{
2499	int rc = VINF_SUCCESS;
2500	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2501	{
2502	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2503	AssertRCReturn(rc, rc);
2504	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2505	}
2506	return rc;
2507	}
2508
2509
2510	/**
2511	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2512	*
2513	* @returns VBox status code.
2514	* @param pVM Pointer to the VM.
2515	* @param pVCpu Pointer to the VMCPU.
2516	* @param pCtx Pointer to the guest-CPU context.
2517	*
2518	* @remarks No-long-jump zone!!!
2519	*/
2520	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2521	{
2522	int rc = VINF_SUCCESS;
2523	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2524	{
2525	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2526	Let us assert it as such and use native-width VMWRITE. */
2527	X86RFLAGS uRFlags = pCtx->rflags;
2528	Assert(uRFlags.u64 >> 32 == 0);
2529	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2530	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2531
2532	/*
2533	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2534	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2535	*/
2536	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2537	{
2538	Assert(pVM->hm.s.vmx.pRealModeTSS);
2539	Assert(PDMVmmDevHeapIsEnabled(pVM));
2540	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2541	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2542	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2543	}
2544
2545	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2546	AssertRCReturn(rc, rc);
2547
2548	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2549	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2550	}
2551	return rc;
2552	}
2553
2554
2555	/**
2556	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2557	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2558	* assembly code.
2559	*
2560	* @returns VBox status code.
2561	* @param pVM Pointer to the VM.
2562	* @param pVCpu Pointer to the VMCPU.
2563	* @param pCtx Pointer to the guest-CPU context.
2564	*
2565	* @remarks No-long-jump zone!!!
2566	*/
2567	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2568	{
2569	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2570	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2571	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2572	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2573	AssertRCReturn(rc, rc);
2574	return rc;
2575	}
2576
2577
2578	/**
2579	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2580	* in the VMCS.
2581	*
2582	* @returns VBox status code.
2583	* @param pVM Pointer to the VM.
2584	* @param pVCpu Pointer to the VMCPU.
2585	* @param pCtx Pointer to the guest-CPU context.
2586	*
2587	* @remarks No-long-jump zone!!!
2588	*/
2589	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2590	{
2591	int rc = VINF_SUCCESS;
2592
2593	/*
2594	* Guest CR0.
2595	* Guest FPU.
2596	*/
2597	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2598	{
2599	uint64_t u64GuestCR0 = pCtx->cr0;
2600
2601	/* The guest's view (read access) of its CR0 is unblemished. */
2602	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2603	AssertRCReturn(rc, rc);
2604	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2605
2606	/* Setup VT-x's view of the guest CR0. */
2607	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2608	if (pVM->hm.s.fNestedPaging)
2609	{
2610	if (CPUMIsGuestPagingEnabledEx(pCtx))
2611	{
2612	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2613	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2614	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2615	}
2616	else
2617	{
2618	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2619	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2620	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2621	}
2622
2623	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2624	AssertRCReturn(rc, rc);
2625	}
2626	else
2627	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2628
2629	/*
2630	* Guest FPU bits.
2631	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2632	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2633	*/
2634	u64GuestCR0 \|= X86_CR0_NE;
2635	bool fInterceptNM = false;
2636	if (CPUMIsGuestFPUStateActive(pVCpu))
2637	{
2638	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2639	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2640	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2641	}
2642	else
2643	{
2644	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2645	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2646	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2647	}
2648
2649	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2650	bool fInterceptMF = false;
2651	if (!(pCtx->cr0 & X86_CR0_NE))
2652	fInterceptMF = true;
2653
2654	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2655	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2656	{
2657	Assert(PDMVmmDevHeapIsEnabled(pVM));
2658	Assert(pVM->hm.s.vmx.pRealModeTSS);
2659	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2660	fInterceptNM = true;
2661	fInterceptMF = true;
2662	}
2663	else
2664	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2665
2666	if (fInterceptNM)
2667	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2668	else
2669	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2670
2671	if (fInterceptMF)
2672	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2673	else
2674	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2675
2676	/* Additional intercepts for debugging, define these yourself explicitly. */
2677	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2678	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2679	\| RT_BIT(X86_XCPT_DB)
2680	\| RT_BIT(X86_XCPT_DE)
2681	\| RT_BIT(X86_XCPT_NM)
2682	\| RT_BIT(X86_XCPT_UD)
2683	\| RT_BIT(X86_XCPT_NP)
2684	\| RT_BIT(X86_XCPT_SS)
2685	\| RT_BIT(X86_XCPT_GP)
2686	\| RT_BIT(X86_XCPT_PF)
2687	\| RT_BIT(X86_XCPT_MF);
2688	#elif defined(VBOX_ALWAYS_TRAP_PF)
2689	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2690	#endif
2691
2692	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2693
2694	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2695	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2696	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2697	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2698	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2699	else
2700	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2701
2702	u64GuestCR0 \|= uSetCR0;
2703	u64GuestCR0 &= uZapCR0;
2704	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2705
2706	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2707	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2708	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2709	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2710
2711	/*
2712	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2713	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2714	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2715	*/
2716	uint64_t u64CR0Mask = 0;
2717	u64CR0Mask = X86_CR0_PE
2718	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2719	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2720	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2721	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2722	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2723
2724	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2725	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2726	u64CR0Mask &= ~X86_CR0_PE;
2727
2728	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2729	if (fInterceptNM)
2730	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2731	else
2732	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2733
2734	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2735	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2736	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2737	AssertRCReturn(rc, rc);
2738
2739	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2740	}
2741
2742	/*
2743	* Guest CR2.
2744	* It's always loaded in the assembler code. Nothing to do here.
2745	*/
2746
2747	/*
2748	* Guest CR3.
2749	*/
2750	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2751	{
2752	uint64_t u64GuestCR3 = 0;
2753	if (pVM->hm.s.fNestedPaging)
2754	{
2755	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2756
2757	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2758	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2759	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2760	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2761
2762	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2763	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2764	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2765
2766	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2767	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2768	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2769	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2770
2771	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2772	AssertRCReturn(rc, rc);
2773	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2774
2775	if ( CPUMIsGuestPagingEnabledEx(pCtx)
2776	\|\| pVM->hm.s.vmx.fUnrestrictedGuest)
2777	{
2778	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2779	if (CPUMIsGuestInPAEModeEx(pCtx))
2780	{
2781	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2782	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2783	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2784	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2785	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2786	AssertRCReturn(rc, rc);
2787	}
2788
2789	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2790	have Unrestricted Execution to handle the guest when it's not using paging. */
2791	u64GuestCR3 = pCtx->cr3;
2792	}
2793	else
2794	{
2795	/*
2796	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2797	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2798	* EPT takes care of translating it to host-physical addresses.
2799	*/
2800	RTGCPHYS GCPhys;
2801	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2802	Assert(PDMVmmDevHeapIsEnabled(pVM));
2803
2804	/* We obtain it here every time as the guest could have relocated this PCI region. */
2805	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2806	AssertRCReturn(rc, rc);
2807
2808	u64GuestCR3 = GCPhys;
2809	}
2810	}
2811	else
2812	{
2813	/* Non-nested paging case, just use the hypervisor's CR3. */
2814	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2815	}
2816
2817	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2818	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2819	AssertRCReturn(rc, rc);
2820
2821	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2822	}
2823
2824	/*
2825	* Guest CR4.
2826	*/
2827	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2828	{
2829	uint64_t u64GuestCR4 = pCtx->cr4;
2830
2831	/* The guest's view of its CR4 is unblemished. */
2832	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2833	AssertRCReturn(rc, rc);
2834	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2835
2836	/* Setup VT-x's view of the guest CR4. */
2837	/*
2838	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2839	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2840	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2841	*/
2842	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2843	{
2844	Assert(pVM->hm.s.vmx.pRealModeTSS);
2845	Assert(PDMVmmDevHeapIsEnabled(pVM));
2846	u64GuestCR4 &= ~X86_CR4_VME;
2847	}
2848
2849	if (pVM->hm.s.fNestedPaging)
2850	{
2851	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2852	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2853	{
2854	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2855	u64GuestCR4 \|= X86_CR4_PSE;
2856	/* Our identity mapping is a 32 bits page directory. */
2857	u64GuestCR4 &= ~X86_CR4_PAE;
2858	}
2859	/* else use guest CR4.*/
2860	}
2861	else
2862	{
2863	/*
2864	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2865	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2866	*/
2867	switch (pVCpu->hm.s.enmShadowMode)
2868	{
2869	case PGMMODE_REAL: /* Real-mode. */
2870	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2871	case PGMMODE_32_BIT: /* 32-bit paging. */
2872	{
2873	u64GuestCR4 &= ~X86_CR4_PAE;
2874	break;
2875	}
2876
2877	case PGMMODE_PAE: /* PAE paging. */
2878	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2879	{
2880	u64GuestCR4 \|= X86_CR4_PAE;
2881	break;
2882	}
2883
2884	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2885	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2886	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2887	break;
2888	#endif
2889	default:
2890	AssertFailed();
2891	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2892	}
2893	}
2894
2895	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2896	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2897	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2898	u64GuestCR4 \|= uSetCR4;
2899	u64GuestCR4 &= uZapCR4;
2900
2901	/* Write VT-x's view of the guest CR4 into the VMCS. */
2902	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2903	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2904
2905	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2906	uint64_t u64CR4Mask = 0;
2907	u64CR4Mask = X86_CR4_VME
2908	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2909	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2910	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2911	\| X86_CR4_VMXE;
2912	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2913	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2914	AssertRCReturn(rc, rc);
2915
2916	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2917	}
2918	return rc;
2919	}
2920
2921
2922	/**
2923	* Loads the guest debug registers into the guest-state area in the VMCS.
2924	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2925	*
2926	* @returns VBox status code.
2927	* @param pVM Pointer to the VM.
2928	* @param pVCpu Pointer to the VMCPU.
2929	* @param pCtx Pointer to the guest-CPU context.
2930	*
2931	* @remarks No-long-jump zone!!!
2932	*/
2933	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2934	{
2935	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2936	return VINF_SUCCESS;
2937
2938	#ifdef DEBUG
2939	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2940	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2941	{
2942	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2943	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2944	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2945	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2946	}
2947	#endif
2948
2949	int rc = VERR_INTERNAL_ERROR_5;
2950	bool fInterceptDB = false;
2951	bool fInterceptMovDRx = false;
2952	if (DBGFIsStepping(pVCpu))
2953	{
2954	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2955	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2956	{
2957	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2958	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2959	AssertRCReturn(rc, rc);
2960	Assert(fInterceptDB == false);
2961	}
2962	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2963	{
2964	Assert(fInterceptDB == false);
2965	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2966	* just continuing? */
2967	}
2968	else
2969	fInterceptDB = true;
2970	}
2971	else
2972	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2973
2974	/*
2975	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2976	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2977	* The same for the hypervisor DRx registers, priority is for the guest here.
2978	*/
2979	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
2980	&& !CPUMIsGuestDebugStateActive(pVCpu))
2981	{
2982	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
2983	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2984	AssertRCReturn(rc, rc);
2985	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2986	Assert(fInterceptMovDRx == false);
2987	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2988	}
2989	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
2990	&& !CPUMIsHyperDebugStateActive(pVCpu))
2991	{
2992	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
2993	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2994	AssertRCReturn(rc, rc);
2995	Assert(CPUMIsHyperDebugStateActive(pVCpu));
2996	fInterceptMovDRx = true;
2997	}
2998	else
2999	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3000
3001	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3002	if (fInterceptDB)
3003	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3004	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3005	{
3006	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3007	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3008	#endif
3009	}
3010
3011	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3012	if (fInterceptMovDRx)
3013	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3014	else
3015	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3016
3017	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3018	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3019
3020	/* The guest's view of its DR7 is unblemished. */
3021	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3022
3023	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3024	return rc;
3025	}
3026
3027
3028	#ifdef DEBUG
3029	/**
3030	* Debug function to validate segment registers.
3031	*/
3032	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3033	{
3034	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3035	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3036	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3037	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3038	{
3039	/** @todo DPL checks for CS, SS. */
3040	/* Protected mode checks */
3041	/* CS */
3042	Assert(pCtx->cs.Attr.n.u1Present);
3043	Assert(!(pCtx->cs.Attr.u & 0xf00));
3044	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3045	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3046	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3047	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3048	\|\| (pCtx->cs.Attr.n.u1Granularity));
3049	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3050	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3051	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3052	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3053	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3054	else
3055	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3056	/* SS */
3057	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3058	{
3059	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3060	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3061	Assert(pCtx->ss.Attr.n.u1Present);
3062	Assert(!(pCtx->ss.Attr.u & 0xf00));
3063	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3064	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3065	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3066	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3067	\|\| (pCtx->ss.Attr.n.u1Granularity));
3068	}
3069	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3070	/* CR0 might not be up-to-date here always, hence disabled. */
3071	#if 0
3072	if (!pCtx->cr0 & X86_CR0_PE)
3073	Assert(!pCtx->ss.Attr.n.u2Dpl);
3074	#endif
3075	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3076	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3077	{
3078	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3079	Assert(pCtx->ds.Attr.n.u1Present);
3080	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3081	Assert(!(pCtx->ds.Attr.u & 0xf00));
3082	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3083	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3084	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3085	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3086	\|\| (pCtx->ds.Attr.n.u1Granularity));
3087	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3088	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3089	}
3090	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3091	{
3092	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3093	Assert(pCtx->es.Attr.n.u1Present);
3094	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3095	Assert(!(pCtx->es.Attr.u & 0xf00));
3096	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3097	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3098	\|\| !(pCtx->es.Attr.n.u1Granularity));
3099	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3100	\|\| (pCtx->es.Attr.n.u1Granularity));
3101	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3102	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3103	}
3104	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3105	{
3106	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3107	Assert(pCtx->fs.Attr.n.u1Present);
3108	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3109	Assert(!(pCtx->fs.Attr.u & 0xf00));
3110	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3111	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3112	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3113	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3114	\|\| (pCtx->fs.Attr.n.u1Granularity));
3115	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3116	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3117	}
3118	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3119	{
3120	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3121	Assert(pCtx->gs.Attr.n.u1Present);
3122	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3123	Assert(!(pCtx->gs.Attr.u & 0xf00));
3124	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3125	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3126	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3127	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3128	\|\| (pCtx->gs.Attr.n.u1Granularity));
3129	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3130	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3131	}
3132	/* 64-bit capable CPUs. */
3133	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3134	Assert(!(pCtx->cs.u64Base >> 32));
3135	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3136	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3137	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3138	# endif
3139	}
3140	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3141	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3142	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3143	{
3144	/* Real and v86 mode checks. */
3145	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3146	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3147	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3148	{
3149	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3150	}
3151	else
3152	{
3153	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3154	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3155	}
3156
3157	/* CS */
3158	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3159	Assert(pCtx->cs.u32Limit == 0xffff);
3160	Assert(u32CSAttr == 0xf3);
3161	/* SS */
3162	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3163	Assert(pCtx->ss.u32Limit == 0xffff);
3164	Assert(u32SSAttr == 0xf3);
3165	/* DS */
3166	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3167	Assert(pCtx->ds.u32Limit == 0xffff);
3168	Assert(u32DSAttr == 0xf3);
3169	/* ES */
3170	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3171	Assert(pCtx->es.u32Limit == 0xffff);
3172	Assert(u32ESAttr == 0xf3);
3173	/* FS */
3174	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3175	Assert(pCtx->fs.u32Limit == 0xffff);
3176	Assert(u32FSAttr == 0xf3);
3177	/* GS */
3178	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3179	Assert(pCtx->gs.u32Limit == 0xffff);
3180	Assert(u32GSAttr == 0xf3);
3181	/* 64-bit capable CPUs. */
3182	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3183	Assert(!(pCtx->cs.u64Base >> 32));
3184	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3185	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3186	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3187	# endif
3188	}
3189	}
3190	#endif /* DEBUG */
3191
3192
3193	/**
3194	* Writes a guest segment register into the guest-state area in the VMCS.
3195	*
3196	* @returns VBox status code.
3197	* @param pVM Pointer to the VM.
3198	* @param pVCpu Pointer to the VMCPU.
3199	* @param idxSel Index of the selector in the VMCS.
3200	* @param idxLimit Index of the segment limit in the VMCS.
3201	* @param idxBase Index of the segment base in the VMCS.
3202	* @param idxAccess Index of the access rights of the segment in the VMCS.
3203	* @param pSelReg Pointer to the segment selector.
3204	* @param pCtx Pointer to the guest-CPU context.
3205	*
3206	* @remarks No-long-jump zone!!!
3207	*/
3208	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3209	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3210	{
3211	int rc;
3212	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3213	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3214	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3215	AssertRCReturn(rc, rc);
3216
3217	uint32_t u32Access = pSelReg->Attr.u;
3218	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3219	{
3220	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3221	u32Access = 0xf3;
3222	Assert(pVM->hm.s.vmx.pRealModeTSS);
3223	Assert(PDMVmmDevHeapIsEnabled(pVM));
3224	}
3225	else
3226	{
3227	/*
3228	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3229	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3230	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3231	* loaded in protected-mode have their attribute as 0.
3232	*/
3233	if (!u32Access)
3234	u32Access = VMX_SEL_UNUSABLE;
3235	}
3236
3237	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3238	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3239	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3240
3241	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3242	AssertRCReturn(rc, rc);
3243	return rc;
3244	}
3245
3246
3247	/**
3248	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3249	* into the guest-state area in the VMCS.
3250	*
3251	* @returns VBox status code.
3252	* @param pVM Pointer to the VM.
3253	* @param pVCPU Pointer to the VMCPU.
3254	* @param pCtx Pointer to the guest-CPU context.
3255	*
3256	* @remarks No-long-jump zone!!!
3257	* @remarks Requires RFLAGS (for debug assertions).
3258	*/
3259	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3260	{
3261	int rc = VERR_INTERNAL_ERROR_5;
3262
3263	/*
3264	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3265	*/
3266	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3267	{
3268	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3269	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3270	{
3271	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3272	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3273	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3274	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3275	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3276	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3277	}
3278
3279	#ifdef VBOX_WITH_REM
3280	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3281	{
3282	Assert(pVM->hm.s.vmx.pRealModeTSS);
3283	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3284	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3285	{
3286	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3287	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3288	&& enmGuestMode >= PGMMODE_PROTECTED)
3289	{
3290	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3291	in real-mode (e.g. OpenBSD 4.0) */
3292	REMFlushTBs(pVM);
3293	Log2(("Switch to protected mode detected!\n"));
3294	}
3295	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3296	}
3297	}
3298	#endif
3299	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3300	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3301	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3302	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3303	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3304	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3305	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3306	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3307	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3308	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3309	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3310	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3311	AssertRCReturn(rc, rc);
3312
3313	#ifdef DEBUG
3314	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3315	#endif
3316	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3317	}
3318
3319	/*
3320	* Guest TR.
3321	*/
3322	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3323	{
3324	/*
3325	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3326	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3327	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3328	*/
3329	uint16_t u16Sel = 0;
3330	uint32_t u32Limit = 0;
3331	uint64_t u64Base = 0;
3332	uint32_t u32AccessRights = 0;
3333
3334	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3335	{
3336	u16Sel = pCtx->tr.Sel;
3337	u32Limit = pCtx->tr.u32Limit;
3338	u64Base = pCtx->tr.u64Base;
3339	u32AccessRights = pCtx->tr.Attr.u;
3340	}
3341	else
3342	{
3343	Assert(pVM->hm.s.vmx.pRealModeTSS);
3344	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3345
3346	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3347	RTGCPHYS GCPhys;
3348	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3349	AssertRCReturn(rc, rc);
3350
3351	X86DESCATTR DescAttr;
3352	DescAttr.u = 0;
3353	DescAttr.n.u1Present = 1;
3354	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3355
3356	u16Sel = 0;
3357	u32Limit = HM_VTX_TSS_SIZE;
3358	u64Base = GCPhys; /* in real-mode phys = virt. */
3359	u32AccessRights = DescAttr.u;
3360	}
3361
3362	/* Validate. */
3363	Assert(!(u16Sel & RT_BIT(2)));
3364	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3365	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3366	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3367	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3368	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3369	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3370	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3371	Assert( (u32Limit & 0xfff) == 0xfff
3372	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3373	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3374	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3375
3376	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3377	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3378	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3379	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3380	AssertRCReturn(rc, rc);
3381
3382	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3383	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3384	}
3385
3386	/*
3387	* Guest GDTR.
3388	*/
3389	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3390	{
3391	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3392	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3393	AssertRCReturn(rc, rc);
3394
3395	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3396	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3397	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3398	}
3399
3400	/*
3401	* Guest LDTR.
3402	*/
3403	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3404	{
3405	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3406	uint32_t u32Access = 0;
3407	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3408	u32Access = VMX_SEL_UNUSABLE;
3409	else
3410	u32Access = pCtx->ldtr.Attr.u;
3411
3412	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3413	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3414	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3415	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3416	AssertRCReturn(rc, rc);
3417
3418	/* Validate. */
3419	if (!(u32Access & VMX_SEL_UNUSABLE))
3420	{
3421	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3422	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3423	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3424	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3425	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3426	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3427	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3428	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3429	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3430	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3431	}
3432
3433	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3434	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3435	}
3436
3437	/*
3438	* Guest IDTR.
3439	*/
3440	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3441	{
3442	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3443	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3444	AssertRCReturn(rc, rc);
3445
3446	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3447	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3448	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3449	}
3450
3451	/*
3452	* Guest FS & GS base MSRs.
3453	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3454	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3455	* are only available in 64-bit mode.
3456	*/
3457	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3458	* should not be necessary to do it in assembly again. */
3459	if (CPUMIsGuestInLongModeEx(pCtx))
3460	{
3461	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3462	{
3463	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3464	AssertRCReturn(rc, rc);
3465	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3466	}
3467
3468	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3469	{
3470	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3471	AssertRCReturn(rc, rc);
3472	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3473	}
3474	}
3475	else
3476	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3477
3478	return VINF_SUCCESS;
3479	}
3480
3481
3482	/**
3483	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3484	* areas. These MSRs will automatically be loaded to the host CPU on every
3485	* successful VM entry and stored from the host CPU on every successful VM exit.
3486	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3487	*
3488	* @returns VBox status code.
3489	* @param pVM Pointer to the VM.
3490	* @param pVCpu Pointer to the VMCPU.
3491	* @param pCtx Pointer to the guest-CPU context.
3492	*
3493	* @remarks No-long-jump zone!!!
3494	*/
3495	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3496	{
3497	AssertPtr(pVCpu);
3498	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3499
3500	/*
3501	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3502	*/
3503	int rc = VINF_SUCCESS;
3504	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3505	{
3506	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3507	unsigned cGuestMsrs = 0;
3508
3509	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3510	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3511	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3512	if (fSupportsNX \|\| fSupportsLongMode)
3513	{
3514	/** @todo support save IA32_EFER, i.e.
3515	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3516	* guest EFER need not be part of the VM-entry MSR-load area. */
3517	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3518	pGuestMsr->u32Reserved = 0;
3519	pGuestMsr->u64Value = pCtx->msrEFER;
3520	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3521	if (!CPUMIsGuestInLongModeEx(pCtx))
3522	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3523	pGuestMsr++; cGuestMsrs++;
3524	if (fSupportsLongMode)
3525	{
3526	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3527	pGuestMsr->u32Reserved = 0;
3528	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3529	pGuestMsr++; cGuestMsrs++;
3530	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3531	pGuestMsr->u32Reserved = 0;
3532	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3533	pGuestMsr++; cGuestMsrs++;
3534	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3535	pGuestMsr->u32Reserved = 0;
3536	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3537	pGuestMsr++; cGuestMsrs++;
3538	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3539	#if 0
3540	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3541	pGuestMsr->u32Reserved = 0;
3542	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3543	pGuestMsr++; cGuestMsrs++;
3544	#endif
3545	}
3546	}
3547
3548	/*
3549	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3550	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3551	*/
3552	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3553	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3554	{
3555	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3556	pGuestMsr->u32Reserved = 0;
3557	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3558	AssertRCReturn(rc, rc);
3559	pGuestMsr++; cGuestMsrs++;
3560	}
3561
3562	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3563	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3564	{
3565	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3566	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3567	}
3568
3569	/* Update the VCPU's copy of the guest MSR count. */
3570	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3571	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3572	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3573	AssertRCReturn(rc, rc);
3574
3575	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3576	}
3577
3578	/*
3579	* Guest Sysenter MSRs.
3580	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3581	* VM exits on WRMSRs for these MSRs.
3582	*/
3583	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3584	{
3585	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3586	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3587	}
3588	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3589	{
3590	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3591	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3592	}
3593	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3594	{
3595	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3596	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3597	}
3598	AssertRCReturn(rc, rc);
3599
3600	return rc;
3601	}
3602
3603
3604	/**
3605	* Loads the guest activity state into the guest-state area in the VMCS.
3606	*
3607	* @returns VBox status code.
3608	* @param pVM Pointer to the VM.
3609	* @param pVCpu Pointer to the VMCPU.
3610	* @param pCtx Pointer to the guest-CPU context.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3615	{
3616	/** @todo See if we can make use of other states, e.g.
3617	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3618	int rc = VINF_SUCCESS;
3619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3620	{
3621	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3622	AssertRCReturn(rc, rc);
3623	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3624	}
3625	return rc;
3626	}
3627
3628
3629	/**
3630	* Loads the guest-interruptibility state (or "interrupt shadow" as AMD calls
3631	* it) into the guest-state area in the VMCS.
3632	*
3633	* @returns VBox status code.
3634	* @param pVM Pointer to the VM.
3635	* @param pVCpu Pointer to the VMCPU.
3636	* @param pCtx Pointer to the guest-CPU context.
3637	*
3638	* @remarks No-long-jump zone!!!
3639	* @remarks Requires RIP, RFLAGS (debug assert).
3640	*/
3641	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3642	{
3643	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_INTR_STATE))
3644	return VINF_SUCCESS;
3645
3646	/*
3647	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3648	* inhibit interrupts or clear any existing interrupt-inhibition.
3649	*/
3650	uint32_t uIntrState = 0;
3651	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3652	{
3653	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3654	{
3655	/*
3656	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in VT-x
3657	* the flag's condition to be cleared is met and thus the cleared state is correct. Additionally, this means
3658	* we need not re-read the VMCS field on the VM-exit path and clear/set this flag on every VM-exit. Finally,
3659	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
3660	*/
3661	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3662	uIntrState = 0; /* Clear interrupt inhibition. */
3663	}
3664	else if (pCtx->eflags.u32 & X86_EFL_IF)
3665	{
3666	/*
3667	* We don't have enough information to distinguish a block-by-STI vs. block-by-MOV SS. Intel seems to think there
3668	* is a slight difference regarding MOV SS additionally blocking some debug exceptions.
3669	* See Intel spec. 24.2.2 "Guest Non-Register State" table "Format of Interruptibility State".
3670	*/
3671	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3672	}
3673	}
3674	else
3675	uIntrState = 0; /* No interrupt inhibition. */
3676
3677	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3678	Assert((pCtx->eflags.u32 & X86_EFL_IF) \|\| uIntrState == 0); /* If EFLAGS.IF is not set, no interrupt inhibition. */
3679	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3680	AssertRCReturn(rc ,rc);
3681	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_INTR_STATE;
3682	return rc;
3683	}
3684
3685
3686	/**
3687	* Sets up the appropriate function to run guest code.
3688	*
3689	* @returns VBox status code.
3690	* @param pVM Pointer to the VM.
3691	* @param pVCpu Pointer to the VMCPU.
3692	* @param pCtx Pointer to the guest-CPU context.
3693	*
3694	* @remarks No-long-jump zone!!!
3695	* @todo change this to return void.
3696	*/
3697	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3698	{
3699	if (CPUMIsGuestInLongModeEx(pCtx))
3700	{
3701	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3702	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3703	#endif
3704	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3705	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3706	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3707	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3708	#else
3709	/* 64-bit host or hybrid host. */
3710	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3711	#endif
3712	}
3713	else
3714	{
3715	/* Guest is not in long mode, use the 32-bit handler. */
3716	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3717	}
3718	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3719	return VINF_SUCCESS;
3720	}
3721
3722
3723	/**
3724	* Wrapper for running the guest code in VT-x.
3725	*
3726	* @returns VBox strict status code.
3727	* @param pVM Pointer to the VM.
3728	* @param pVCpu Pointer to the VMCPU.
3729	* @param pCtx Pointer to the guest-CPU context.
3730	*
3731	* @remarks No-long-jump zone!!!
3732	*/
3733	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3734	{
3735	/*
3736	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3737	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3738	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3739	*/
3740	#ifdef VBOX_WITH_KERNEL_USING_XMM
3741	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3742	#else
3743	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3744	#endif
3745	}
3746
3747
3748	/**
3749	* Report world-switch error and dump some useful debug info.
3750	*
3751	* @param pVM Pointer to the VM.
3752	* @param pVCpu Pointer to the VMCPU.
3753	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3754	* @param pCtx Pointer to the guest-CPU context.
3755	* @param pVmxTransient Pointer to the VMX transient structure (only
3756	* exitReason updated).
3757	*/
3758	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3759	{
3760	Assert(pVM);
3761	Assert(pVCpu);
3762	Assert(pCtx);
3763	Assert(pVmxTransient);
3764	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3765
3766	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3767	switch (rcVMRun)
3768	{
3769	case VERR_VMX_INVALID_VMXON_PTR:
3770	AssertFailed();
3771	break;
3772	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3773	case VERR_VMX_UNABLE_TO_START_VM:
3774	case VERR_VMX_UNABLE_TO_RESUME_VM:
3775	{
3776	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3777	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3778	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3779	AssertRC(rc);
3780
3781	#ifdef VBOX_STRICT
3782	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3783	pVmxTransient->uExitReason));
3784	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3785	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3786	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3787	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3788	else
3789	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3790
3791	/* VMX control bits. */
3792	uint32_t u32Val;
3793	uint64_t u64Val;
3794	RTHCUINTREG uHCReg;
3795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3796	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3798	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3800	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3802	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3804	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3806	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3808	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3810	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3827	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3828	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3829	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3830	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3831	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3832	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3833	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3834	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3835	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3836	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3837
3838	/* Guest bits. */
3839	RTGCUINTREG uGCReg;
3840	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3841	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3842	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3843	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3844	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3845	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3846	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3847	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3848
3849	/* Host bits. */
3850	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3851	Log(("Host CR0 %#RHr\n", uHCReg));
3852	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3853	Log(("Host CR3 %#RHr\n", uHCReg));
3854	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3855	Log(("Host CR4 %#RHr\n", uHCReg));
3856
3857	RTGDTR HostGdtr;
3858	PCX86DESCHC pDesc;
3859	ASMGetGDTR(&HostGdtr);
3860	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3861	Log(("Host CS %#08x\n", u32Val));
3862	if (u32Val < HostGdtr.cbGdt)
3863	{
3864	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3865	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3866	}
3867
3868	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3869	Log(("Host DS %#08x\n", u32Val));
3870	if (u32Val < HostGdtr.cbGdt)
3871	{
3872	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3873	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3874	}
3875
3876	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3877	Log(("Host ES %#08x\n", u32Val));
3878	if (u32Val < HostGdtr.cbGdt)
3879	{
3880	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3881	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3882	}
3883
3884	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3885	Log(("Host FS %#08x\n", u32Val));
3886	if (u32Val < HostGdtr.cbGdt)
3887	{
3888	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3889	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3890	}
3891
3892	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3893	Log(("Host GS %#08x\n", u32Val));
3894	if (u32Val < HostGdtr.cbGdt)
3895	{
3896	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3897	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3898	}
3899
3900	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3901	Log(("Host SS %#08x\n", u32Val));
3902	if (u32Val < HostGdtr.cbGdt)
3903	{
3904	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3905	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3906	}
3907
3908	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3909	Log(("Host TR %#08x\n", u32Val));
3910	if (u32Val < HostGdtr.cbGdt)
3911	{
3912	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3913	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3914	}
3915
3916	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3917	Log(("Host TR Base %#RHv\n", uHCReg));
3918	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3919	Log(("Host GDTR Base %#RHv\n", uHCReg));
3920	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3921	Log(("Host IDTR Base %#RHv\n", uHCReg));
3922	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3923	Log(("Host SYSENTER CS %#08x\n", u32Val));
3924	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3925	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3926	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3927	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3928	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3929	Log(("Host RSP %#RHv\n", uHCReg));
3930	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3931	Log(("Host RIP %#RHv\n", uHCReg));
3932	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3933	if (VMX_IS_64BIT_HOST_MODE())
3934	{
3935	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3936	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3937	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3938	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3939	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3940	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3941	}
3942	# endif
3943	#endif /* VBOX_STRICT */
3944	break;
3945	}
3946
3947	default:
3948	/* Impossible */
3949	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3950	break;
3951	}
3952	NOREF(pVM);
3953	}
3954
3955
3956	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3957	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3958	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3959	#endif
3960
3961	/**
3962	* Executes the specified handler in 64-bit mode.
3963	*
3964	* @returns VBox status code.
3965	* @param pVM Pointer to the VM.
3966	* @param pVCpu Pointer to the VMCPU.
3967	* @param pCtx Pointer to the guest CPU context.
3968	* @param pfnHandler Pointer to the RC handler function.
3969	* @param cbParam Number of parameters.
3970	* @param paParam Array of 32-bit parameters.
3971	*/
3972	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3973	uint32_t *paParam)
3974	{
3975	int rc, rc2;
3976	PHMGLOBLCPUINFO pCpu;
3977	RTHCPHYS HCPhysCpuPage;
3978	RTHCUINTREG uOldEFlags;
3979
3980	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3981	Assert(pfnHandler);
3982	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3983	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3984
3985	#ifdef VBOX_STRICT
3986	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3987	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3988
3989	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3990	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3991	#endif
3992
3993	/* Disable interrupts. */
3994	uOldEFlags = ASMIntDisableFlags();
3995
3996	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3997	RTCPUID idHostCpu = RTMpCpuId();
3998	CPUMR0SetLApic(pVM, idHostCpu);
3999	#endif
4000
4001	pCpu = HMR0GetCurrentCpu();
4002	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4003
4004	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4005	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4006
4007	/* Leave VMX Root Mode. */
4008	VMXDisable();
4009
4010	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4011
4012	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4013	CPUMSetHyperEIP(pVCpu, pfnHandler);
4014	for (int i = (int)cbParam - 1; i >= 0; i--)
4015	CPUMPushHyper(pVCpu, paParam[i]);
4016
4017	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4018
4019	/* Call the switcher. */
4020	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4021	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4022
4023	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4024	/* Make sure the VMX instructions don't cause #UD faults. */
4025	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4026
4027	/* Re-enter VMX Root Mode */
4028	rc2 = VMXEnable(HCPhysCpuPage);
4029	if (RT_FAILURE(rc2))
4030	{
4031	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4032	ASMSetFlags(uOldEFlags);
4033	return VERR_VMX_VMXON_FAILED;
4034	}
4035
4036	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4037	AssertRC(rc2);
4038	Assert(!(ASMGetFlags() & X86_EFL_IF));
4039	ASMSetFlags(uOldEFlags);
4040	return rc;
4041	}
4042
4043
4044	/**
4045	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4046	* supporting 64-bit guests.
4047	*
4048	* @returns VBox status code.
4049	* @param fResume Whether to VMLAUNCH or VMRESUME.
4050	* @param pCtx Pointer to the guest-CPU context.
4051	* @param pCache Pointer to the VMCS cache.
4052	* @param pVM Pointer to the VM.
4053	* @param pVCpu Pointer to the VMCPU.
4054	*/
4055	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4056	{
4057	uint32_t aParam[6];
4058	PHMGLOBLCPUINFO pCpu = NULL;
4059	RTHCPHYS HCPhysCpuPage = 0;
4060	int rc = VERR_INTERNAL_ERROR_5;
4061
4062	pCpu = HMR0GetCurrentCpu();
4063	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4064
4065	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4066	pCache->uPos = 1;
4067	pCache->interPD = PGMGetInterPaeCR3(pVM);
4068	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4069	#endif
4070
4071	#ifdef DEBUG
4072	pCache->TestIn.HCPhysCpuPage= 0;
4073	pCache->TestIn.HCPhysVmcs = 0;
4074	pCache->TestIn.pCache = 0;
4075	pCache->TestOut.HCPhysVmcs = 0;
4076	pCache->TestOut.pCache = 0;
4077	pCache->TestOut.pCtx = 0;
4078	pCache->TestOut.eflags = 0;
4079	#endif
4080
4081	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4082	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4083	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4084	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4085	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4086	aParam[5] = 0;
4087
4088	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4089	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4090	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4091	#endif
4092	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4093
4094	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4095	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4096	Assert(pCtx->dr[4] == 10);
4097	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4098	#endif
4099
4100	#ifdef DEBUG
4101	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4102	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4103	pVCpu->hm.s.vmx.HCPhysVmcs));
4104	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4105	pCache->TestOut.HCPhysVmcs));
4106	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4107	pCache->TestOut.pCache));
4108	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4109	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4110	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4111	pCache->TestOut.pCtx));
4112	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4113	#endif
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4120	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4121	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4122	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4123	*
4124	* @returns VBox status code.
4125	* @param pVM Pointer to the VM.
4126	* @param pVCpu Pointer to the VMCPU.
4127	*/
4128	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4129	{
4130	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4131	{ \
4132	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4133	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4134	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4135	}
4136
4137	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4138	{ \
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4142	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4143	}
4144
4145	AssertPtr(pVM);
4146	AssertPtr(pVCpu);
4147	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4148
4149	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4150	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4151	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4152	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4153	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4154	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4155	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4156	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4157	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4158
4159	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4160	#if 0
4161	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4162	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4166	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4167	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4169	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4170	#endif
4171
4172	/* 32-bit guest-state fields. */
4173	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4174	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4175	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4176	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4177	/* Unused 32-bit guest-state fields. */
4178	#if 0
4179	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4180	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4181	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4182	#endif
4183
4184	/* Natural width guest-state fields. */
4185	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4186	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4187	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4188	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4189	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4190	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4191	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4192	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4193	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4194	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4195	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4196	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4197	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4198	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4199	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4200	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4201	/* Unused natural width guest-state fields. */
4202	#if 0
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4204	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4205	#endif
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4208
4209	if (pVM->hm.s.fNestedPaging)
4210	{
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4213	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4214	}
4215	else
4216	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4217
4218	#undef VMXLOCAL_INIT_VMCS_SELREG
4219	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4220	return VINF_SUCCESS;
4221	}
4222
4223
4224	/**
4225	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4226	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4227	* darwin, running 64-bit guests).
4228	*
4229	* @returns VBox status code.
4230	* @param pVCpu Pointer to the VMCPU.
4231	* @param idxField The VMCS field encoding.
4232	* @param u64Val 16, 32 or 64 bits value.
4233	*/
4234	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4235	{
4236	int rc;
4237	switch (idxField)
4238	{
4239	/*
4240	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4241	*/
4242	/* 64-bit Control fields. */
4243	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4244	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4245	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4246	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4247	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4248	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4249	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4250	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4251	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4252	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4253	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4254	case VMX_VMCS64_CTRL_EPTP_FULL:
4255	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4256	/* 64-bit Read-only data fields. */
4257	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4258	/* 64-bit Guest-state fields. */
4259	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4260	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4261	case VMX_VMCS64_GUEST_PAT_FULL:
4262	case VMX_VMCS64_GUEST_EFER_FULL:
4263	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4264	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4265	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4266	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4267	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4268	/* 64-bit Host-state fields. */
4269	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4270	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4271	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4272	{
4273	rc = VMXWriteVmcs32(idxField, u64Val);
4274	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4275	break;
4276	}
4277
4278	/*
4279	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4280	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4281	*/
4282	/* Natural-width Guest-state fields. */
4283	case VMX_VMCS_GUEST_CR0:
4284	case VMX_VMCS_GUEST_CR3:
4285	case VMX_VMCS_GUEST_CR4:
4286	case VMX_VMCS_GUEST_ES_BASE:
4287	case VMX_VMCS_GUEST_CS_BASE:
4288	case VMX_VMCS_GUEST_SS_BASE:
4289	case VMX_VMCS_GUEST_DS_BASE:
4290	case VMX_VMCS_GUEST_FS_BASE:
4291	case VMX_VMCS_GUEST_GS_BASE:
4292	case VMX_VMCS_GUEST_LDTR_BASE:
4293	case VMX_VMCS_GUEST_TR_BASE:
4294	case VMX_VMCS_GUEST_GDTR_BASE:
4295	case VMX_VMCS_GUEST_IDTR_BASE:
4296	case VMX_VMCS_GUEST_DR7:
4297	case VMX_VMCS_GUEST_RSP:
4298	case VMX_VMCS_GUEST_RIP:
4299	case VMX_VMCS_GUEST_RFLAGS:
4300	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4301	case VMX_VMCS_GUEST_SYSENTER_ESP:
4302	case VMX_VMCS_GUEST_SYSENTER_EIP:
4303	{
4304	if ((u64Val >> 32ULL) == 0)
4305	{
4306	/* If this field is 64-bit, VT-x will zero out the top bits. */
4307	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4308	}
4309	else
4310	{
4311	/* Assert that only the 32->64 switcher case should ever come here. */
4312	Assert(pVM->hm.s.fAllow64BitGuests);
4313	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4314	}
4315	break;
4316	}
4317
4318	default:
4319	{
4320	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4321	rc = VERR_INVALID_PARAMETER;
4322	break;
4323	}
4324	}
4325	AssertRCReturn(rc, rc);
4326	return rc;
4327	}
4328
4329
4330	/**
4331	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4332	* hosts (except darwin) for 64-bit guests.
4333	*
4334	* @param pVCpu Pointer to the VMCPU.
4335	* @param idxField The VMCS field encoding.
4336	* @param u64Val 16, 32 or 64 bits value.
4337	*/
4338	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4339	{
4340	AssertPtr(pVCpu);
4341	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4342
4343	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4344	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4345
4346	/* Make sure there are no duplicates. */
4347	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4348	{
4349	if (pCache->Write.aField[i] == idxField)
4350	{
4351	pCache->Write.aFieldVal[i] = u64Val;
4352	return VINF_SUCCESS;
4353	}
4354	}
4355
4356	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4357	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4358	pCache->Write.cValidEntries++;
4359	return VINF_SUCCESS;
4360	}
4361
4362
4363	/**
4364	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4365	*
4366	* @param pCache Pointer to the VMCS cache.
4367	*/
4368	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4369	{
4370	AssertPtr(pCache);
4371	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4372	{
4373	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4374	AssertRC(rc, rc);
4375	}
4376	pCache->Write.cValidEntries = 0;
4377	}
4378
4379
4380	/**
4381	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4382	*
4383	* @param pCache Pointer to the VMCS cache.
4384	* @remarks No-long-jump zone!!!
4385	*/
4386	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4387	{
4388	AssertPtr(pCache);
4389	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4390	{
4391	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4392	AssertRC(rc);
4393	}
4394	}
4395	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4396
4397
4398	/**
4399	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4400	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4401	* timer.
4402	*
4403	* @returns VBox status code.
4404	* @param pVM Pointer to the VM.
4405	* @param pVCpu Pointer to the VMCPU.
4406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4407	* out-of-sync. Make sure to update the required fields
4408	* before using them.
4409	* @remarks No-long-jump zone!!!
4410	*/
4411	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4412	{
4413	int rc = VERR_INTERNAL_ERROR_5;
4414	bool fOffsettedTsc = false;
4415	if (pVM->hm.s.vmx.fUsePreemptTimer)
4416	{
4417	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4418
4419	/* Make sure the returned values have sane upper and lower boundaries. */
4420	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4421	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4422	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4423	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4424
4425	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4426	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4427	}
4428	else
4429	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4430
4431	if (fOffsettedTsc)
4432	{
4433	uint64_t u64CurTSC = ASMReadTSC();
4434	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4435	{
4436	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4437	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4438
4439	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4440	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4441	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4442	}
4443	else
4444	{
4445	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4446	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4447	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4448	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4449	}
4450	}
4451	else
4452	{
4453	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4454	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4455	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4456	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4457	}
4458	}
4459
4460
4461	/**
4462	* Determines if an exception is a benign exception. Benign exceptions
4463	* are ones which cannot cause double-faults.
4464	*
4465	* @returns true if the exception is benign, false otherwise.
4466	* @param uVector The exception vector.
4467	*/
4468	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint8_t uVector)
4469	{
4470	switch (uVector)
4471	{
4472	case X86_XCPT_DB:
4473	case X86_XCPT_NMI:
4474	case X86_XCPT_BP:
4475	case X86_XCPT_OF:
4476	case X86_XCPT_BR:
4477	case X86_XCPT_UD:
4478	case X86_XCPT_NM:
4479	case X86_XCPT_CO_SEG_OVERRUN:
4480	case X86_XCPT_MF:
4481	case X86_XCPT_AC:
4482	case X86_XCPT_MC:
4483	case X86_XCPT_XF:
4484	return true;
4485	default:
4486	return false;
4487	}
4488	}
4489
4490
4491	/**
4492	* Determines if an exception is a contributory exception. Contributory
4493	* exceptions are ones which can cause double-faults.
4494	*
4495	* @returns true if the exception is contributory, false otherwise.
4496	* @param uVector The exception vector.
4497	*/
4498	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint8_t uVector)
4499	{
4500	switch (uVector)
4501	{
4502	case X86_XCPT_GP:
4503	case X86_XCPT_SS:
4504	case X86_XCPT_NP:
4505	case X86_XCPT_TS:
4506	case X86_XCPT_DE:
4507	return true;
4508	default:
4509	return false;
4510	}
4511	return false;
4512	}
4513
4514
4515	/**
4516	* Determines if we are intercepting any contributory exceptions.
4517	*
4518	* @returns true if we are intercepting any contributory exception, false
4519	* otherwise.
4520	* @param pVCpu Pointer to the VMCPU.
4521	*/
4522	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4523	{
4524	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4525	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4526	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4527	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4528	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4529	{
4530	return true;
4531	}
4532	return false;
4533	}
4534
4535
4536	/**
4537	* Handle a condition that occurred while delivering an event through the guest
4538	* IDT.
4539	*
4540	* @returns VBox status code (informational error codes included).
4541	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4542	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4543	* continue execution of the guest which will delivery the #DF.
4544	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4545	*
4546	* @param pVM Pointer to the VM.
4547	* @param pVCpu Pointer to the VMCPU.
4548	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4549	* out-of-sync. Make sure to update the required fields
4550	* before using them.
4551	* @param pVmxTransient Pointer to the VMX transient structure.
4552	*
4553	* @remarks No-long-jump zone!!!
4554	*/
4555	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4556	{
4557	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4558	AssertRCReturn(rc, rc);
4559	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4560	{
4561	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4562	AssertRCReturn(rc, rc);
4563
4564	uint8_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4565	uint8_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4566	uint8_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4567
4568	typedef enum
4569	{
4570	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4571	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4572	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4573	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4574	} VMXREFLECTXCPT;
4575
4576	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4577	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4578	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4579	{
4580	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4581	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4582	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4583	&& uExitVector == X86_XCPT_PF))
4584	{
4585	enmReflect = VMXREFLECTXCPT_XCPT;
4586	}
4587	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4588	&& hmR0VmxIsContributoryXcpt(uExitVector))
4589	{
4590	enmReflect = VMXREFLECTXCPT_DF;
4591	}
4592	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4593	&& uIdtVector == X86_XCPT_PF
4594	&& uExitVector == X86_XCPT_PF)
4595	{
4596	pVmxTransient->fVectoringPF = true;
4597	}
4598	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4599	&& uIdtVector == X86_XCPT_PF
4600	&& hmR0VmxIsContributoryXcpt(uExitVector))
4601	{
4602	enmReflect = VMXREFLECTXCPT_DF;
4603	}
4604	else if (uIdtVector == X86_XCPT_DF)
4605	enmReflect = VMXREFLECTXCPT_TF;
4606	}
4607	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4608	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4609	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4610	{
4611	/*
4612	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4613	* (whatever they are) as they reoccur when restarting the instruction.
4614	*/
4615	enmReflect = VMXREFLECTXCPT_XCPT;
4616	}
4617
4618	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4619	switch (enmReflect)
4620	{
4621	case VMXREFLECTXCPT_XCPT:
4622	{
4623	Assert(!pVCpu->hm.s.Event.fPending);
4624	pVCpu->hm.s.Event.fPending = true;
4625	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4626	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4627	{
4628	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4629	AssertRCReturn(rc, rc);
4630	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4631	}
4632	else
4633	pVCpu->hm.s.Event.u32ErrCode = 0;
4634	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4635	break;
4636	}
4637
4638	case VMXREFLECTXCPT_DF:
4639	{
4640	uint32_t u32IntrInfo;
4641	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4642	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4643	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4644
4645	Assert(!pVCpu->hm.s.Event.fPending);
4646	pVCpu->hm.s.Event.fPending = true;
4647	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4648	pVCpu->hm.s.Event.u32ErrCode = 0;
4649	rc = VINF_VMX_DOUBLE_FAULT;
4650	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4651	break;
4652	}
4653
4654	case VMXREFLECTXCPT_TF:
4655	{
4656	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4657	rc = VINF_EM_RESET;
4658	break;
4659	}
4660
4661	default: /* shut up gcc. */
4662	break;
4663	}
4664	}
4665	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4666	return rc;
4667	}
4668
4669
4670	/**
4671	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4672	*
4673	* @returns VBox status code.
4674	* @param pVM Pointer to the VM.
4675	* @param pVCpu Pointer to the VMCPU.
4676	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4677	* out-of-sync. Make sure to update the required fields
4678	* before using them.
4679	*
4680	* @remarks No-long-jump zone!!!
4681	*/
4682	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4683	{
4684	int rc = VINF_SUCCESS;
4685	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4686	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4687	{
4688	RTGCUINTREG uVal = 0;
4689	RTCCUINTREG uShadow = 0;
4690	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4691	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4692	AssertRCReturn(rc, rc);
4693	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4694	CPUMSetGuestCR0(pVCpu, uVal);
4695	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4696	}
4697	return rc;
4698	}
4699
4700
4701	/**
4702	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4703	*
4704	* @returns VBox status code.
4705	* @param pVM Pointer to the VM.
4706	* @param pVCpu Pointer to the VMCPU.
4707	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4708	* out-of-sync. Make sure to update the required fields
4709	* before using them.
4710	*
4711	* @remarks No-long-jump zone!!!
4712	*/
4713	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4714	{
4715	int rc = VINF_SUCCESS;
4716	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4717	{
4718	RTGCUINTREG uVal = 0;
4719	RTCCUINTREG uShadow = 0;
4720	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4721	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4722	AssertRCReturn(rc, rc);
4723	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4724	CPUMSetGuestCR4(pVCpu, uVal);
4725	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4726	}
4727	return rc;
4728	}
4729
4730
4731	/**
4732	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4733	*
4734	* @returns VBox status code.
4735	* @param pVM Pointer to the VM.
4736	* @param pVCpu Pointer to the VMCPU.
4737	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4738	* out-of-sync. Make sure to update the required fields
4739	* before using them.
4740	*
4741	* @remarks No-long-jump zone!!!
4742	*/
4743	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4744	{
4745	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4746	return VINF_SUCCESS;
4747
4748	RTGCUINTREG uVal = 0;
4749	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4750	AssertRCReturn(rc, rc);
4751	pMixedCtx->rip = uVal;
4752	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4753	return rc;
4754	}
4755
4756
4757	/**
4758	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4759	*
4760	* @returns VBox status code.
4761	* @param pVM Pointer to the VM.
4762	* @param pVCpu Pointer to the VMCPU.
4763	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4764	* out-of-sync. Make sure to update the required fields
4765	* before using them.
4766	*
4767	* @remarks No-long-jump zone!!!
4768	*/
4769	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4770	{
4771	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4772	return VINF_SUCCESS;
4773
4774	RTGCUINTREG uVal = 0;
4775	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4776	AssertRCReturn(rc, rc);
4777	pMixedCtx->rsp = uVal;
4778	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4779	return rc;
4780	}
4781
4782
4783	/**
4784	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4785	*
4786	* @returns VBox status code.
4787	* @param pVM Pointer to the VM.
4788	* @param pVCpu Pointer to the VMCPU.
4789	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4790	* out-of-sync. Make sure to update the required fields
4791	* before using them.
4792	*
4793	* @remarks No-long-jump zone!!!
4794	*/
4795	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4796	{
4797	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4798	return VINF_SUCCESS;
4799
4800	RTGCUINTREG uVal = 0;
4801	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4802	AssertRCReturn(rc, rc);
4803	pMixedCtx->rflags.u64 = uVal;
4804
4805	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4806	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4807	{
4808	Assert(pVM->hm.s.vmx.pRealModeTSS);
4809	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4810	pMixedCtx->eflags.Bits.u1VM = 0;
4811	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4812	}
4813
4814	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4815	return rc;
4816	}
4817
4818
4819	/**
4820	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4821	* guest-CPU context.
4822	*/
4823	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824	{
4825	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4826	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4827	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4828	AssertRCReturn(rc, rc);
4829	return rc;
4830	}
4831
4832
4833	/**
4834	* Saves the guest's interruptibility state.
4835	*
4836	* @returns VBox status code.
4837	* @param pVM Pointer to the VM.
4838	* @param pVCpu Pointer to the VMCPU.
4839	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4840	* out-of-sync. Make sure to update the required fields
4841	* before using them.
4842	*
4843	* @remarks No-long-jump zone!!!
4844	*/
4845	DECLINLINE(int) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4846	{
4847	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_INTR_STATE)
4848	return VINF_SUCCESS;
4849
4850	uint32_t uIntrState = 0;
4851	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4852	if (uIntrState != 0)
4853	{
4854	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4855	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4856	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4857	AssertRCReturn(rc, rc);
4858	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4859	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4860	}
4861	else
4862	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4863
4864	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_INTR_STATE;
4865	return rc;
4866	}
4867
4868
4869	/**
4870	* Saves the guest's activity state.
4871	*
4872	* @returns VBox status code.
4873	* @param pVM Pointer to the VM.
4874	* @param pVCpu Pointer to the VMCPU.
4875	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4876	* out-of-sync. Make sure to update the required fields
4877	* before using them.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882	{
4883	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4884	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4885	return VINF_SUCCESS;
4886	}
4887
4888
4889	/**
4890	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4891	* the current VMCS into the guest-CPU context.
4892	*
4893	* @returns VBox status code.
4894	* @param pVM Pointer to the VM.
4895	* @param pVCpu Pointer to the VMCPU.
4896	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4897	* out-of-sync. Make sure to update the required fields
4898	* before using them.
4899	*
4900	* @remarks No-long-jump zone!!!
4901	*/
4902	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4903	{
4904	int rc = VINF_SUCCESS;
4905	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4906	{
4907	uint32_t u32Val = 0;
4908	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4909	pMixedCtx->SysEnter.cs = u32Val;
4910	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4911	}
4912
4913	RTGCUINTREG uGCVal = 0;
4914	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4915	{
4916	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4917	pMixedCtx->SysEnter.eip = uGCVal;
4918	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4919	}
4920	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4921	{
4922	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4923	pMixedCtx->SysEnter.esp = uGCVal;
4924	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4925	}
4926	return rc;
4927	}
4928
4929
4930	/**
4931	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4932	* context.
4933	*
4934	* @returns VBox status code.
4935	* @param pVM Pointer to the VM.
4936	* @param pVCpu Pointer to the VMCPU.
4937	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4938	* out-of-sync. Make sure to update the required fields
4939	* before using them.
4940	*
4941	* @remarks No-long-jump zone!!!
4942	*/
4943	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4944	{
4945	RTGCUINTREG uVal = 0;
4946	int rc = VINF_SUCCESS;
4947	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4948	{
4949	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4950	pMixedCtx->fs.u64Base = uVal;
4951	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4952	}
4953	return rc;
4954	}
4955
4956
4957	/**
4958	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4959	* context.
4960	*
4961	* @returns VBox status code.
4962	* @param pVM Pointer to the VM.
4963	* @param pVCpu Pointer to the VMCPU.
4964	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4965	* out-of-sync. Make sure to update the required fields
4966	* before using them.
4967	*
4968	* @remarks No-long-jump zone!!!
4969	*/
4970	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4971	{
4972	RTGCUINTREG uVal = 0;
4973	int rc = VINF_SUCCESS;
4974	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4975	{
4976	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4977	pMixedCtx->gs.u64Base = uVal;
4978	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4979	}
4980	return rc;
4981	}
4982
4983
4984	/**
4985	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4986	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4987	*
4988	* @returns VBox status code.
4989	* @param pVM Pointer to the VM.
4990	* @param pVCpu Pointer to the VMCPU.
4991	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4992	* out-of-sync. Make sure to update the required fields
4993	* before using them.
4994	*
4995	* @remarks No-long-jump zone!!!
4996	*/
4997	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4998	{
4999	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5000	return VINF_SUCCESS;
5001
5002	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5003	{
5004	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5005	pMsr += i;
5006	switch (pMsr->u32IndexMSR)
5007	{
5008	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5009	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5010	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5011	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5012	#if 0
5013	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5014	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5015	#endif
5016	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5017	default:
5018	{
5019	AssertFailed();
5020	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5021	}
5022	}
5023	}
5024	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5025	return VINF_SUCCESS;
5026	}
5027
5028
5029	/**
5030	* Saves the guest control registers from the current VMCS into the guest-CPU
5031	* context.
5032	*
5033	* @returns VBox status code.
5034	* @param pVM Pointer to the VM.
5035	* @param pVCpu Pointer to the VMCPU.
5036	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5037	* out-of-sync. Make sure to update the required fields
5038	* before using them.
5039	*
5040	* @remarks No-long-jump zone!!!
5041	*/
5042	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5043	{
5044	RTGCUINTREG uVal = 0;
5045	RTGCUINTREG uShadow = 0;
5046	int rc = VINF_SUCCESS;
5047
5048	/* Guest CR0. Guest FPU. */
5049	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5050	AssertRCReturn(rc, rc);
5051
5052	/* Guest CR4. */
5053	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5054	AssertRCReturn(rc, rc);
5055
5056	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5057	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5058	{
5059	if ( pVM->hm.s.fNestedPaging
5060	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5061	{
5062	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5063	AssertRCReturn(rc, rc);
5064	if (pMixedCtx->cr3 != uVal)
5065	{
5066	CPUMSetGuestCR3(pVCpu, uVal);
5067	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5068	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5069	}
5070
5071	/* We require EFER to check PAE mode. */
5072	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5073	AssertRCReturn(rc, rc);
5074
5075	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5076	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5077	{
5078	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5079	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5080	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5081	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5082	AssertRCReturn(rc, rc);
5083	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5084	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5085	}
5086	}
5087	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5088	}
5089	return rc;
5090	}
5091
5092
5093	/**
5094	* Reads a guest segment register from the current VMCS into the guest-CPU
5095	* context.
5096	*
5097	* @returns VBox status code.
5098	* @param idxSel Index of the selector in the VMCS.
5099	* @param idxLimit Index of the segment limit in the VMCS.
5100	* @param idxBase Index of the segment base in the VMCS.
5101	* @param idxAccess Index of the access rights of the segment in the VMCS.
5102	* @param pSelReg Pointer to the segment selector.
5103	*
5104	* @remarks No-long-jump zone!!!
5105	*/
5106	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5107	PCPUMSELREG pSelReg)
5108	{
5109	uint32_t u32Val = 0;
5110	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5111	pSelReg->Sel = (uint16_t)u32Val;
5112	pSelReg->ValidSel = (uint16_t)u32Val;
5113	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5114
5115	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5116	pSelReg->u32Limit = u32Val;
5117
5118	RTGCUINTREG uGCVal = 0;
5119	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5120	pSelReg->u64Base = uGCVal;
5121
5122	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5123	pSelReg->Attr.u = u32Val;
5124
5125	/*
5126	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5127	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5128	*/
5129	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5130	{
5131	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5132	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5133	}
5134	return rc;
5135	}
5136
5137
5138	/**
5139	* Saves the guest segment registers from the current VMCS into the guest-CPU
5140	* context.
5141	*
5142	* @returns VBox status code.
5143	* @param pVM Pointer to the VM.
5144	* @param pVCpu Pointer to the VMCPU.
5145	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5146	* out-of-sync. Make sure to update the required fields
5147	* before using them.
5148	*
5149	* @remarks No-long-jump zone!!!
5150	*/
5151	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5152	{
5153	int rc = VINF_SUCCESS;
5154
5155	/* Guest segment registers. */
5156	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5157	{
5158	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5159	AssertRCReturn(rc, rc);
5160
5161	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5162	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5163	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5164	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5165	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5166	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5167	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5168	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5169	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5170	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5171	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5172	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5173	AssertRCReturn(rc, rc);
5174
5175	/* Restore segment attributes for real-on-v86 mode hack. */
5176	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5177	{
5178	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5179	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5180	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5181	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5182	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5183	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5184	}
5185
5186	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5187	}
5188
5189	/* Guest LDTR. */
5190	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5191	{
5192	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5193	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5194	AssertRCReturn(rc, rc);
5195	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5196	}
5197
5198	/* Guest GDTR. */
5199	RTGCUINTREG uGCVal = 0;
5200	uint32_t u32Val = 0;
5201	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5202	{
5203	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5204	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5205	pMixedCtx->gdtr.pGdt = uGCVal;
5206	pMixedCtx->gdtr.cbGdt = u32Val;
5207	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5208	}
5209
5210	/* Guest IDTR. */
5211	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5212	{
5213	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5214	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5215	pMixedCtx->idtr.pIdt = uGCVal;
5216	pMixedCtx->idtr.cbIdt = u32Val;
5217	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5218	}
5219
5220	/* Guest TR. */
5221	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5222	{
5223	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5224	AssertRCReturn(rc, rc);
5225
5226	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5227	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5228	{
5229	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5230	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5231	AssertRCReturn(rc, rc);
5232	}
5233	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5234	}
5235	return rc;
5236	}
5237
5238
5239	/**
5240	* Saves the guest debug registers from the current VMCS into the guest-CPU
5241	* context.
5242	*
5243	* @returns VBox status code.
5244	* @param pVM Pointer to the VM.
5245	* @param pVCpu Pointer to the VMCPU.
5246	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5247	* out-of-sync. Make sure to update the required fields
5248	* before using them.
5249	*
5250	* @remarks No-long-jump zone!!!
5251	*/
5252	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5253	{
5254	int rc = VINF_SUCCESS;
5255	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5256	{
5257	RTGCUINTREG uVal;
5258	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5259	pMixedCtx->dr[7] = uVal;
5260
5261	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5262	}
5263	return rc;
5264	}
5265
5266
5267	/**
5268	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5269	*
5270	* @returns VBox status code.
5271	* @param pVM Pointer to the VM.
5272	* @param pVCpu Pointer to the VMCPU.
5273	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5274	* out-of-sync. Make sure to update the required fields
5275	* before using them.
5276	*
5277	* @remarks No-long-jump zone!!!
5278	*/
5279	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5280	{
5281	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5282	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5283	return VINF_SUCCESS;
5284	}
5285
5286
5287	/**
5288	* Saves the entire guest state from the currently active VMCS into the
5289	* guest-CPU context. This essentially VMREADs all guest-data.
5290	*
5291	* @returns VBox status code.
5292	* @param pVM Pointer to the VM.
5293	* @param pVCpu Pointer to the VMCPU.
5294	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5295	* out-of-sync. Make sure to update the required fields
5296	* before using them.
5297	*/
5298	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5299	{
5300	Assert(pVM);
5301	Assert(pVCpu);
5302	Assert(pMixedCtx);
5303
5304	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5305	return VINF_SUCCESS;
5306
5307	VMMRZCallRing3Disable(pVCpu); /* We're not using hmR0VmxCallRing3Disable() as we want to respect nesting here. */
5308
5309	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5311
5312	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5314
5315	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5317
5318	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5320
5321	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5323
5324	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5326
5327	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5329
5330	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5332
5333	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
5334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestIntrState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5335
5336	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5338
5339	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5341
5342	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5343	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5344
5345	VMMRZCallRing3Enable(pVCpu);
5346	return rc;
5347	}
5348
5349
5350	/**
5351	* Check per-VM and per-VCPU force flag actions that require us to go back to
5352	* ring-3 for one reason or another.
5353	*
5354	* @returns VBox status code (information status code included).
5355	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5356	* ring-3.
5357	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5358	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5359	* interrupts)
5360	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5361	* all EMTs to be in ring-3.
5362	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5363	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5364	* to the EM loop.
5365	*
5366	* @param pVM Pointer to the VM.
5367	* @param pVCpu Pointer to the VMCPU.
5368	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5369	* out-of-sync. Make sure to update the required fields
5370	* before using them.
5371	*/
5372	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5373	{
5374	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5375
5376	int rc = VERR_INTERNAL_ERROR_5;
5377	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5378	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5379	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5380	{
5381	/* We need the control registers now, make sure the guest-CPU context is updated. */
5382	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5383	AssertRCReturn(rc, rc);
5384
5385	/* Pending HM CR3 sync. */
5386	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5387	{
5388	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5389	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5390	}
5391	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5392	{
5393	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5394	AssertRCReturn(rc, rc);
5395	}
5396
5397	/* Pending PGM C3 sync. */
5398	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5399	{
5400	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5401	if (rc != VINF_SUCCESS)
5402	{
5403	AssertRC(rc);
5404	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5405	return rc;
5406	}
5407	}
5408
5409	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5410	/* -XXX- what was that about single stepping? */
5411	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5412	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5413	{
5414	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5415	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5416	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5417	return rc;
5418	}
5419
5420	/* Pending VM request packets, such as hardware interrupts. */
5421	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5422	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5423	{
5424	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5425	return VINF_EM_PENDING_REQUEST;
5426	}
5427
5428	/* Pending PGM pool flushes. */
5429	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5430	{
5431	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5432	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5433	}
5434
5435	/* Pending DMA requests. */
5436	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5437	{
5438	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5439	return VINF_EM_RAW_TO_R3;
5440	}
5441	}
5442
5443	/* Paranoia. */
5444	Assert(rc != VERR_EM_INTERPRETER);
5445	return VINF_SUCCESS;
5446	}
5447
5448
5449	/**
5450	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5451	* VT-x to execute any instruction.
5452	*
5453	* @param pvCpu Pointer to the VMCPU.
5454	*/
5455	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5456	{
5457	if (pVCpu->hm.s.Event.fPending)
5458	{
5459	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5460	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5461	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5462	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5463
5464	/* If a trap was already pending, we did something wrong! */
5465	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5466
5467	/* A page-fault exception during a page-fault would become a double-fault. */
5468	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5469	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5470
5471	TRPMEVENT enmTrapType;
5472	switch (uVectorType)
5473	{
5474	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5475	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5476	enmTrapType = TRPM_HARDWARE_INT;
5477	break;
5478	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5479	enmTrapType = TRPM_SOFTWARE_INT;
5480	break;
5481	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5482	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5483	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5484	enmTrapType = TRPM_TRAP;
5485	break;
5486	default:
5487	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5488	enmTrapType = TRPM_32BIT_HACK;
5489	break;
5490	}
5491	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5492	AssertRC(rc);
5493	if (fErrorCodeValid)
5494	TRPMSetErrorCode(pVCpu, uErrorCode);
5495
5496	/* Clear the VT-x state bits now that TRPM has the information. */
5497	pVCpu->hm.s.Event.fPending = false;
5498	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5499	AssertRC(rc);
5500	}
5501	}
5502
5503
5504	/**
5505	* Does the necessary state syncing before doing a longjmp to ring-3.
5506	*
5507	* @param pVM Pointer to the VM.
5508	* @param pVCpu Pointer to the VMCPU.
5509	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5510	* out-of-sync. Make sure to update the required fields
5511	* before using them.
5512	* @param rcExit The reason for exiting to ring-3. Can be
5513	* VINF_VMM_UNKNOWN_RING3_CALL.
5514	*
5515	* @remarks No-long-jmp zone!!!
5516	*/
5517	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5518	{
5519	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5520	Log(("hmR0VmxLongJmpToRing3: rcExit=%d\n", rcExit));
5521
5522	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5523	AssertRC(rc);
5524
5525	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5526	if (CPUMIsGuestFPUStateActive(pVCpu))
5527	{
5528	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5529	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5531	}
5532
5533	/* Restore debug registers if necessary and resync on next R0 reentry. */
5534	if (CPUMIsGuestDebugStateActive(pVCpu))
5535	{
5536	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5537	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5538	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5539	}
5540	else if (CPUMIsHyperDebugStateActive(pVCpu))
5541	{
5542	CPUMR0LoadHostDebugState(pVM, pVCpu);
5543	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5544	}
5545
5546	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5547	}
5548
5549
5550	/**
5551	* An action requires us to go back to ring-3. This function does the necessary
5552	* steps before we can safely return to ring-3. This is not the same as longjmps
5553	* to ring-3, this is voluntary.
5554	*
5555	* @param pVM Pointer to the VM.
5556	* @param pVCpu Pointer to the VMCPU.
5557	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5558	* out-of-sync. Make sure to update the required fields
5559	* before using them.
5560	* @param rcExit The reason for exiting to ring-3. Can be
5561	* VINF_VMM_UNKNOWN_RING3_CALL.
5562	*/
5563	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5564	{
5565	Assert(pVM);
5566	Assert(pVCpu);
5567	Assert(pMixedCtx);
5568	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5569
5570	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5571	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5572	return;
5573
5574	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5575	VMMRZCallRing3Disable(pVCpu);
5576	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5577
5578	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5579	hmR0VmxUpdateTRPMTrap(pVCpu);
5580
5581	/* Sync. the guest state. */
5582	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5583	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5584
5585	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5586	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5587	\| CPUM_CHANGED_LDTR
5588	\| CPUM_CHANGED_GDTR
5589	\| CPUM_CHANGED_IDTR
5590	\| CPUM_CHANGED_TR
5591	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5592
5593	/* On our way back from ring-3 the following needs to be done. */
5594	/** @todo This can change with preemption hooks. */
5595	if (rcExit == VINF_EM_RAW_INTERRUPT)
5596	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5597	else
5598	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5599
5600	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5601	VMMRZCallRing3Enable(pVCpu);
5602	}
5603
5604
5605	/**
5606	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5607	* longjump to ring-3 and possibly get preempted.
5608	*
5609	* @param pVCpu Pointer to the VMCPU.
5610	* @param enmOperation The operation causing the ring-3 longjump.
5611	* @param pvUser The user argument (pointer to the possibly
5612	* out-of-date guest-CPU context).
5613	*
5614	* @remarks Must never be called with @a enmOperation ==
5615	* VMMCALLRING3_VM_R0_ASSERTION.
5616	*/
5617	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5618	{
5619	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5620	Assert(pVCpu);
5621	Assert(pvUser);
5622	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5623	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5624
5625	VMMRZCallRing3Disable(pVCpu);
5626	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5627	VMMRZCallRing3Enable(pVCpu);
5628	}
5629
5630
5631	/**
5632	* Injects any pending TRPM trap into the VM by updating the VMCS.
5633	*
5634	* @returns VBox status code (informational status code included).
5635	* @param pVM Pointer to the VM.
5636	* @param pVCpu Pointer to the VMCPU.
5637	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5638	* out-of-sync. Make sure to update the required fields
5639	* before using them.
5640	*/
5641	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5642	{
5643	if (!TRPMHasTrap(pVCpu))
5644	return VINF_SUCCESS;
5645
5646	uint8_t u8Vector = 0;
5647	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5648	RTGCUINT uErrCode = 0;
5649
5650	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5651	AssertRCReturn(rc, rc);
5652	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5653
5654	rc = TRPMResetTrap(pVCpu);
5655	AssertRCReturn(rc, rc);
5656
5657	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5658	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5659	if (enmTrpmEvent == TRPM_TRAP)
5660	{
5661	switch (u8Vector)
5662	{
5663	case X86_XCPT_BP:
5664	case X86_XCPT_OF:
5665	{
5666	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5667	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5668	break;
5669	}
5670
5671	case X86_XCPT_DF:
5672	case X86_XCPT_TS:
5673	case X86_XCPT_NP:
5674	case X86_XCPT_SS:
5675	case X86_XCPT_GP:
5676	case X86_XCPT_PF:
5677	case X86_XCPT_AC:
5678	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5679	them which VT-x/VMM pushes to the guest stack. */
5680	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5681	/* no break! */
5682	default:
5683	{
5684	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5685	break;
5686	}
5687	}
5688	}
5689	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5690	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5691	else
5692	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5693
5694	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5695	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5696	}
5697
5698
5699	/**
5700	* Checks if there are any pending guest interrupts to be delivered and injects
5701	* them into the VM by updating the VMCS.
5702	*
5703	* @returns VBox status code (informational status codes included).
5704	* @param pVM Pointer to the VM.
5705	* @param pVCpu Pointer to the VMCPU.
5706	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5707	* out-of-sync. Make sure to update the required fields
5708	* before using them.
5709	*
5710	* @remarks This must be called only after hmR0VmxSaveGuestIntrState().
5711	*/
5712	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5713	{
5714	/* First inject any pending HM interrupts. */
5715	if (pVCpu->hm.s.Event.fPending)
5716	{
5717	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5718	pVCpu->hm.s.Event.u32ErrCode);
5719	AssertRCReturn(rc, rc);
5720	pVCpu->hm.s.Event.fPending = false;
5721	return rc;
5722	}
5723
5724	/** @todo SMI. SMIs take priority over NMIs. */
5725
5726	/* NMI. NMIs take priority over regular interrupts . */
5727	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5728	{
5729	/* Construct an NMI interrupt and inject it into the VMCS. */
5730	RTGCUINTPTR uIntrInfo;
5731	uIntrInfo = X86_XCPT_NMI;
5732	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5733	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5734	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5735	AssertRCReturn(rc, rc);
5736	return rc;
5737	}
5738
5739	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5740	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5741	AssertRCReturn(rc, rc);
5742
5743	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5744	if (!TRPMHasTrap(pVCpu))
5745	{
5746	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5747	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5748	{
5749	/*
5750	* When external interrupts are pending and the guest has disabled interrupts, cause a VM-exit using "interrupt-window
5751	* exiting" so we can deliver the interrupt when the guest is ready to receive them. Otherwise, if the guest
5752	* can receive interrupts now, convert the PDM interrupt into a TRPM event and inject it.
5753	*/
5754	if (!(pMixedCtx->eflags.u32 & X86_EFL_IF)) /** @todo we can use interrupt-window exiting for block-by-STI. */
5755	{
5756	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5757	{
5758	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5759	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5760	AssertRCReturn(rc, rc);
5761	}
5762	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5763	}
5764	else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5765	{
5766	uint8_t u8Interrupt = 0;
5767	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5768	if (RT_SUCCESS(rc))
5769	{
5770	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5771	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5772	AssertRCReturn(rc, rc);
5773	}
5774	else
5775	{
5776	/** @todo Does this actually happen? If not turn it into an assertion. */
5777	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5778	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5779	}
5780	}
5781	}
5782	}
5783
5784	/* If interrupts can be delivered, inject it into the VM. */
5785	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5786	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5787	&& TRPMHasTrap(pVCpu))
5788	{
5789	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5790	AssertRCReturn(rc, rc);
5791	}
5792	return rc;
5793	}
5794
5795	/**
5796	* Injects an invalid-opcode (#UD) exception into the VM.
5797	*
5798	* @returns VBox status code (informational status code included).
5799	* @param pVM Pointer to the VM.
5800	* @param pVCpu Pointer to the VMCPU.
5801	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5802	* out-of-sync. Make sure to update the required fields
5803	* before using them.
5804	*/
5805	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5806	{
5807	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5808	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5809	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5810	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5811	}
5812
5813
5814	/**
5815	* Injects a double-fault (#DF) exception into the VM.
5816	*
5817	* @returns VBox status code (informational status code included).
5818	* @param pVM Pointer to the VM.
5819	* @param pVCpu Pointer to the VMCPU.
5820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5821	* out-of-sync. Make sure to update the required fields
5822	* before using them.
5823	*/
5824	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5825	{
5826	/* Inject the double-fault. */
5827	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5828	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5829	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5830	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5831	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5832	}
5833
5834
5835	/**
5836	* Injects a debug (#DB) exception into the VM.
5837	*
5838	* @returns VBox status code (informational status code included).
5839	* @param pVM Pointer to the VM.
5840	* @param pVCpu Pointer to the VMCPU.
5841	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5842	* out-of-sync. Make sure to update the required fields
5843	* before using them.
5844	*/
5845	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5846	{
5847	/* Inject the debug-exception. */
5848	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5849	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5850	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5851	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5852	}
5853
5854
5855	/**
5856	* Injects a overflow (#OF) exception into the VM.
5857	*
5858	* @returns VBox status code (informational status code included).
5859	* @param pVM Pointer to the VM.
5860	* @param pVCpu Pointer to the VMCPU.
5861	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5862	* out-of-sync. Make sure to update the required fields
5863	* before using them.
5864	* @param cbInstr The value of RIP that is to be pushed on the guest
5865	* stack.
5866	*/
5867	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5868	{
5869	/* Inject the overflow exception. */
5870	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5871	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5872	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5873	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5874	}
5875
5876
5877	/**
5878	* Injects a general-protection (#GP) fault into the VM.
5879	*
5880	* @returns VBox status code (informational status code included).
5881	* @param pVM Pointer to the VM.
5882	* @param pVCpu Pointer to the VMCPU.
5883	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5884	* out-of-sync. Make sure to update the required fields
5885	* before using them.
5886	* @param u32ErrorCode The error code associated with the #GP.
5887	*/
5888	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5889	{
5890	/* Inject the general-protection fault. */
5891	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5892	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5893	if (fErrorCodeValid)
5894	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5895	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5896	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5897	}
5898
5899
5900	/**
5901	* Injects a software interrupt (INTn) into the VM.
5902	*
5903	* @returns VBox status code (informational status code included).
5904	* @param pVM Pointer to the VM.
5905	* @param pVCpu Pointer to the VMCPU.
5906	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5907	* out-of-sync. Make sure to update the required fields
5908	* before using them.
5909	* @param uVector The software interrupt vector number.
5910	* @param cbInstr The value of RIP that is to be pushed on the guest
5911	* stack.
5912	*/
5913	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5914	{
5915	/* Inject the INTn. */
5916	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5917	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5918	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5919	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5920	}
5921
5922
5923	/**
5924	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5925	* stack.
5926	*
5927	* @returns VBox status code (information status code included).
5928	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5929	* @param pVM Pointer to the VM.
5930	* @param pMixedCtx Pointer to the guest-CPU context.
5931	* @param uValue The value to push to the guest stack.
5932	*/
5933	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5934	{
5935	/*
5936	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5937	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5938	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5939	*/
5940	if (pMixedCtx->sp == 1)
5941	return VINF_EM_RESET;
5942	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5943	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5944	AssertRCReturn(rc, rc);
5945	return rc;
5946	}
5947
5948
5949	/**
5950	* Injects an event into the guest upon VM-entry by updating the relevant fields
5951	* in the VM-entry area in the VMCS.
5952	*
5953	* @returns VBox status code (informational error codes included).
5954	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5955	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5956	*
5957	* @param pVM Pointer to the VM.
5958	* @param pVCpu Pointer to the VMCPU.
5959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5960	* out-of-sync. Make sure to update the required fields
5961	* before using them.
5962	* @param u64IntrInfo The VM-entry interruption-information field.
5963	* @param cbInstr The VM-entry instruction length in bytes (for software
5964	* interrupts, exceptions and privileged software
5965	* exceptions).
5966	* @param u32ErrCode The VM-entry exception error code.
5967	*
5968	* @remarks No-long-jump zone!!!
5969	*/
5970	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5971	uint32_t u32ErrCode)
5972	{
5973	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5974	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5975	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5976
5977	/* We require CR0 to check if the guest is in real-mode. */
5978	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5979	AssertRCReturn(rc, rc);
5980
5981	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5982	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5983
5984	/*
5985	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5986	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5987	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5988	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5989	*/
5990	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5991	{
5992	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5993	{
5994	Assert(PDMVmmDevHeapIsEnabled(pVM));
5995	Assert(pVM->hm.s.vmx.pRealModeTSS);
5996
5997	/* Save the required guest state bits from the VMCS. */
5998	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5999	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6000	AssertRCReturn(rc, rc);
6001
6002	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6003	const size_t cbIdtEntry = 4;
6004	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6005	{
6006	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6007	if (uVector == X86_XCPT_DF)
6008	return VINF_EM_RESET;
6009	else if (uVector == X86_XCPT_GP)
6010	{
6011	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6012	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6013	}
6014
6015	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6016	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6017	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6018	}
6019
6020	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6021	uint16_t uGuestIp = pMixedCtx->ip;
6022	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6023	{
6024	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6025	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6026	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6027	}
6028	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6029	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6030
6031	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6032	uint16_t offIdtEntry = 0;
6033	RTSEL selIdtEntry = 0;
6034	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6035	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6036	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6037	AssertRCReturn(rc, rc);
6038
6039	/* Construct the stack frame for the interrupt/exception handler. */
6040	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6041	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6042	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6043	AssertRCReturn(rc, rc);
6044
6045	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6046	if (rc == VINF_SUCCESS)
6047	{
6048	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6049	pMixedCtx->rip = offIdtEntry;
6050	pMixedCtx->cs.Sel = selIdtEntry;
6051	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6052
6053	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6054	\| HM_CHANGED_GUEST_RIP
6055	\| HM_CHANGED_GUEST_RFLAGS
6056	\| HM_CHANGED_GUEST_RSP;
6057	AssertRCReturn(rc, rc);
6058	}
6059	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6060	return rc;
6061	}
6062	else
6063	{
6064	/*
6065	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6066	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6067	*/
6068	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6069	}
6070	}
6071
6072	/* Add the valid bit, maybe the caller was lazy. */
6073	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6074
6075	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6076	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6077	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6078
6079	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6080	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6081	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6082	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6083	AssertRCReturn(rc, rc);
6084
6085	return rc;
6086	}
6087
6088
6089	/**
6090	* Enters the VT-x session.
6091	*
6092	* @returns VBox status code.
6093	* @param pVM Pointer to the VM.
6094	* @param pVCpu Pointer to the VMCPU.
6095	* @param pCpu Pointer to the CPU info struct.
6096	*/
6097	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6098	{
6099	AssertPtr(pVM);
6100	AssertPtr(pVCpu);
6101	Assert(pVM->hm.s.vmx.fSupported);
6102	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6103	NOREF(pCpu);
6104
6105	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6106
6107	/* Make sure we're in VMX root mode. */
6108	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6109	if (!(u32HostCR4 & X86_CR4_VMXE))
6110	{
6111	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6112	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6113	}
6114
6115	/* Load the active VMCS as the current one. */
6116	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6117	if (RT_FAILURE(rc))
6118	return rc;
6119
6120	/** @todo this will change with preemption hooks where can can VMRESUME as long
6121	* as we're no preempted. */
6122	pVCpu->hm.s.fResumeVM = false;
6123	return VINF_SUCCESS;
6124	}
6125
6126
6127	/**
6128	* Leaves the VT-x session.
6129	*
6130	* @returns VBox status code.
6131	* @param pVM Pointer to the VM.
6132	* @param pVCpu Pointer to the VMCPU.
6133	* @param pCtx Pointer to the guest-CPU context.
6134	*/
6135	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6136	{
6137	AssertPtr(pVCpu);
6138	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6139	NOREF(pVM);
6140	NOREF(pCtx);
6141
6142	/** @todo this will change with preemption hooks where we only VMCLEAR when
6143	* we are actually going to be preempted, not all the time like we
6144	* currently do. */
6145	/*
6146	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6147	* and mark the VMCS launch-state as "clear".
6148	*/
6149	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6150	return rc;
6151	}
6152
6153
6154	/**
6155	* Saves the host state in the VMCS host-state.
6156	* Sets up the VM-exit MSR-load area.
6157	*
6158	* The CPU state will be loaded from these fields on every successful VM-exit.
6159	*
6160	* @returns VBox status code.
6161	* @param pVM Pointer to the VM.
6162	* @param pVCpu Pointer to the VMCPU.
6163	*
6164	* @remarks No-long-jump zone!!!
6165	*/
6166	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6167	{
6168	AssertPtr(pVM);
6169	AssertPtr(pVCpu);
6170	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6171
6172	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6173
6174	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6175	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6176	return VINF_SUCCESS;
6177
6178	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6179	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6180
6181	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6182	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6183
6184	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6185	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6186
6187	/* Reset the host-state-changed flag. */
6188	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6189	return rc;
6190	}
6191
6192
6193	/**
6194	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6195	* loaded from these fields on every successful VM-entry.
6196	*
6197	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6198	* Sets up the VM-entry controls.
6199	* Sets up the appropriate VMX non-root function to execute guest code based on
6200	* the guest CPU mode.
6201	*
6202	* @returns VBox status code.
6203	* @param pVM Pointer to the VM.
6204	* @param pVCpu Pointer to the VMCPU.
6205	* @param pCtx Pointer to the guest-CPU context.
6206	*
6207	* @remarks No-long-jump zone!!!
6208	*/
6209	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6210	{
6211	AssertPtr(pVM);
6212	AssertPtr(pVCpu);
6213	AssertPtr(pCtx);
6214	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6215
6216	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6217
6218	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6219
6220	/* Determine real-on-v86 mode. */
6221	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6222	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6223	&& CPUMIsGuestInRealModeEx(pCtx))
6224	{
6225	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6226	}
6227
6228	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6229	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6230
6231	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6232	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6233
6234	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6235	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6236
6237	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6238	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6239
6240	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6241	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6242
6243	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6244	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6245
6246	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6247	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6248
6249	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6250	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6251
6252	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6253	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6254
6255	rc = hmR0VmxLoadGuestIntrState(pVM, pVCpu, pCtx);
6256	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestIntrState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6257
6258	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6259	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6260
6261	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6262	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6263	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6264
6265	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6266	return rc;
6267	}
6268
6269
6270	/**
6271	* Does the preparations before executing guest code in VT-x.
6272	*
6273	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6274	* recompiler. We must be cautious what we do here regarding committing
6275	* guest-state information into the the VMCS assuming we assuredly execute the
6276	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6277	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6278	* that the recompiler can (and should) use them when it resumes guest
6279	* execution. Otherwise such operations must be done when we can no longer
6280	* exit to ring-3.
6281	*
6282	* @returns VBox status code (informational status codes included).
6283	* @retval VINF_SUCCESS if we can proceed with running the guest.
6284	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6285	* into the guest.
6286	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6287	*
6288	* @param pVM Pointer to the VM.
6289	* @param pVCpu Pointer to the VMCPU.
6290	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6291	* out-of-sync. Make sure to update the required fields
6292	* before using them.
6293	* @param pVmxTransient Pointer to the VMX transient structure.
6294	*
6295	* @remarks Called with preemption disabled.
6296	*/
6297	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6298	{
6299	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6300
6301	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6302	PGMRZDynMapFlushAutoSet(pVCpu);
6303	#endif
6304
6305	/* Check force flag actions that might require us to go back to ring-3. */
6306	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6307	if (rc != VINF_SUCCESS)
6308	return rc;
6309
6310	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6311	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6312	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6313	{
6314	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6315	RTGCPHYS GCPhysApicBase;
6316	GCPhysApicBase = pMixedCtx->msrApicBase;
6317	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6318
6319	/* Unalias any existing mapping. */
6320	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6321	AssertRCReturn(rc, rc);
6322
6323	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6324	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6325	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6326	AssertRCReturn(rc, rc);
6327
6328	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6329	}
6330
6331	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6332	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6333	pVmxTransient->uEFlags = ASMIntDisableFlags();
6334	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6335	{
6336	ASMSetFlags(pVmxTransient->uEFlags);
6337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
6338	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6339	return VINF_EM_RAW_INTERRUPT;
6340	}
6341	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6342	#endif
6343
6344	/*
6345	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6346	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6347	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6348	*/
6349	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6350	AssertRCReturn(rc, rc);
6351	return rc;
6352	}
6353
6354
6355	/**
6356	* Prepares to run guest code in VT-x and we've committed to doing so. This
6357	* means there is no backing out to ring-3 or anywhere else at this
6358	* point.
6359	*
6360	* @param pVM Pointer to the VM.
6361	* @param pVCpu Pointer to the VMCPU.
6362	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6363	* out-of-sync. Make sure to update the required fields
6364	* before using them.
6365	* @param pVmxTransient Pointer to the VMX transient structure.
6366	*
6367	* @remarks Called with preemption disabled.
6368	* @remarks No-long-jump zone!!!
6369	*/
6370	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6371	{
6372	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6373
6374	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6375	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6376	pVmxTransient->uEFlags = ASMIntDisableFlags();
6377	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6378	#endif
6379
6380	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6381	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6382	int rc;
6383	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_INTR_STATE)
6384	rc = hmR0VmxLoadGuestIntrState(pVM, pVCpu, pMixedCtx);
6385	else
6386	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6387	AssertRC(rc);
6388	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6389
6390	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6391	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6392	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6393
6394	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6395	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6396	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6397
6398	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6399	if (pVmxTransient->fUpdateTscOffsetting)
6400	{
6401	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6402	pVmxTransient->fUpdateTscOffsetting = false;
6403	}
6404
6405	/*
6406	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6407	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6408	*/
6409	if (pVM->hm.s.fTPRPatchingActive)
6410	{
6411	Assert(!CPUMIsGuestInLongMode(pVCpu));
6412
6413	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6414	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6415	AssertRC(rc);
6416
6417	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6418	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6419	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6420	see hmR0VmxLoadGuestApicState(). */
6421	}
6422
6423	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6424	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6425	to start executing. */
6426	}
6427
6428
6429	/**
6430	* Performs some essential restoration of state after running guest code in
6431	* VT-x.
6432	*
6433	* @param pVM Pointer to the VM.
6434	* @param pVCpu Pointer to the VMCPU.
6435	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6436	* out-of-sync. Make sure to update the required fields
6437	* before using them.
6438	* @param pVmxTransient Pointer to the VMX transient structure.
6439	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6440	*
6441	* @remarks Called with interrupts disabled.
6442	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6443	* unconditionally when it is safe to do so.
6444	*/
6445	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6446	{
6447	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6448	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6449
6450	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6451	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6452	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6453	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6454	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6455
6456	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6457	{
6458	/** @todo Find a way to fix hardcoding a guestimate. */
6459	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6460	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6461	}
6462
6463	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6464	Assert(!(ASMGetFlags() & X86_EFL_IF));
6465	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6466
6467	/* Restore the effects of TPR patching if any. */
6468	if (pVM->hm.s.fTPRPatchingActive)
6469	{
6470	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6471	AssertRC(rc);
6472	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6473	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6474	}
6475
6476	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6477	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6478
6479	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6480	uint32_t uExitReason;
6481	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6482	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6483	AssertRC(rc);
6484	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6485	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6486
6487	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6488	hmR0VmxCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6489
6490	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6491	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6492	{
6493	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6494	return;
6495	}
6496
6497	/* We need to update our interruptibility-state on every VM-exit and VM-entry. */
6498	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6499	AssertRC(rc);
6500	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE;
6501
6502	/*
6503	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever we
6504	* eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why we do
6505	* it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6506	*/
6507	if ( !pVmxTransient->fVMEntryFailed
6508	&& (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6509	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6510	{
6511	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6512	AssertRC(rc);
6513	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6514	}
6515	}
6516
6517
6518	/**
6519	* Runs the guest code using VT-x.
6520	*
6521	* @returns VBox status code.
6522	* @param pVM Pointer to the VM.
6523	* @param pVCpu Pointer to the VMCPU.
6524	* @param pCtx Pointer to the guest-CPU context.
6525	*
6526	* @remarks Called with preemption disabled.
6527	*/
6528	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6529	{
6530	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6531	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6532
6533	VMXTRANSIENT VmxTransient;
6534	VmxTransient.fUpdateTscOffsetting = true;
6535	int rc = VERR_INTERNAL_ERROR_5;
6536	unsigned cLoops = 0;
6537
6538	for (;; cLoops++)
6539	{
6540	Assert(!HMR0SuspendPending());
6541	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6542	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6543	(unsigned)RTMpCpuId(), cLoops));
6544
6545	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6546	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6547	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6548	if (rc != VINF_SUCCESS)
6549	break;
6550
6551	/*
6552	* No longjmps to ring-3 from this point on!!!
6553	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6554	* This also disables flushing of the R0-logger instance (if any).
6555	*/
6556	hmR0VmxCallRing3Disable(pVCpu);
6557	VMMRZCallRing3RemoveNotification(pVCpu);
6558	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6559
6560	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6561	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6562
6563	/*
6564	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6565	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6566	*/
6567	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6568	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6569	{
6570	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6571	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6572	return rc;
6573	}
6574
6575	/* Handle the VM-exit. */
6576	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6577	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6578	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6579	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6580	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6581	if (rc != VINF_SUCCESS)
6582	break;
6583	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6584	{
6585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6586	rc = VINF_EM_RAW_INTERRUPT;
6587	break;
6588	}
6589	}
6590
6591	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6592	if (rc == VERR_EM_INTERPRETER)
6593	rc = VINF_EM_RAW_EMULATE_INSTR;
6594	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6595	return rc;
6596	}
6597
6598	#ifdef DEBUG
6599	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6600	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6601	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6602	# define VMX_ASSERT_PREEMPT_CPUID() \
6603	do \
6604	{ \
6605	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6606	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6607	} while (0)
6608
6609	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6610	do { \
6611	AssertPtr(pVM); \
6612	AssertPtr(pVCpu); \
6613	AssertPtr(pMixedCtx); \
6614	AssertPtr(pVmxTransient); \
6615	Assert(pVmxTransient->fVMEntryFailed == false); \
6616	Assert(ASMIntAreEnabled() == true); \
6617	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6618	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6619	LogFunc(("\n")); \
6620	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6621	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6622	VMX_ASSERT_PREEMPT_CPUID(); \
6623	} while (0)
6624	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6625	do { \
6626	LogFunc(("\n")); \
6627	} while(0)
6628	#else /* Release builds */
6629	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6630	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6631	#endif
6632
6633
6634	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6635	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6636	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6637	/**
6638	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6639	*/
6640	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6641	{
6642	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6643	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6644	return VINF_SUCCESS;
6645	}
6646
6647
6648	/**
6649	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6650	*/
6651	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6652	{
6653	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6654	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6655	AssertRCReturn(rc, rc);
6656
6657	uint8_t u8IntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6658	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6659	&& u8IntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6660
6661	if (u8IntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6662	return VINF_EM_RAW_INTERRUPT;
6663
6664	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6665	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(uExitIntrInfo));
6666
6667	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6668	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6669	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6670	return VINF_SUCCESS;
6671	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6672	return rc;
6673
6674	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6675	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo))
6676	{
6677	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6678	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6679	/* no break */
6680	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6681	{
6682	switch (uVector)
6683	{
6684	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6691	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6692	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6693	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6694	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6695	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6696	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6697	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6698	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6699	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6700	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6701	#endif
6702	default:
6703	{
6704	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6705	AssertRCReturn(rc, rc);
6706
6707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6708	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6709	{
6710	Assert(pVM->hm.s.vmx.pRealModeTSS);
6711	Assert(PDMVmmDevHeapIsEnabled(pVM));
6712	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6713	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6714	AssertRCReturn(rc, rc);
6715	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6716	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6717	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6718	AssertRCReturn(rc, rc);
6719	}
6720	else
6721	{
6722	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6723	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6724	}
6725	break;
6726	}
6727	}
6728	break;
6729	}
6730
6731	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6732	default:
6733	{
6734	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6735	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6736	break;
6737	}
6738	}
6739	return rc;
6740	}
6741
6742
6743	/**
6744	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6745	*/
6746	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6747	{
6748	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6749	int rc = VERR_INTERNAL_ERROR_5;
6750	#ifdef DEBUG
6751	rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6752	AssertRC(rc);
6753	Assert(pMixedCtx->eflags.u32 & X86_EFL_IF);
6754	#endif
6755
6756	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6757	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6758	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6759	AssertRCReturn(rc, rc);
6760
6761	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6762	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6763	return VINF_SUCCESS;
6764	}
6765
6766
6767	/**
6768	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6769	*/
6770	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6771	{
6772	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6773	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6774	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6775	}
6776
6777
6778	/**
6779	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6780	*/
6781	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6782	{
6783	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6784	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6785	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6786	AssertRCReturn(rc, rc);
6787
6788	pMixedCtx->rip += pVmxTransient->cbInstr;
6789	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6790
6791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6792	return VINF_SUCCESS;
6793	}
6794
6795
6796	/**
6797	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6798	*/
6799	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6800	{
6801	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6802	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6803	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6804	AssertRCReturn(rc, rc);
6805
6806	pMixedCtx->rip += pVmxTransient->cbInstr;
6807	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6808
6809	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6810	return VINF_SUCCESS;
6811	}
6812
6813
6814	/**
6815	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6816	*/
6817	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6818	{
6819	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6820	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6821	if (RT_LIKELY(rc == VINF_SUCCESS))
6822	{
6823	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6824	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6825	AssertRCReturn(rc, rc);
6826	Assert(pVmxTransient->cbInstr == 2);
6827
6828	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6829	pMixedCtx->rip += pVmxTransient->cbInstr;
6830	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6831	}
6832	else
6833	{
6834	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6835	rc = VERR_EM_INTERPRETER;
6836	}
6837	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6838	return rc;
6839	}
6840
6841
6842	/**
6843	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6844	*/
6845	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6846	{
6847	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6848	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6849	AssertRCReturn(rc, rc);
6850
6851	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6852	return VINF_EM_RAW_EMULATE_INSTR;
6853
6854	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6855	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6856	}
6857
6858
6859	/**
6860	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6861	*/
6862	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6863	{
6864	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6865	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6866	AssertRCReturn(rc, rc);
6867
6868	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6869	if (RT_LIKELY(rc == VINF_SUCCESS))
6870	{
6871	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6872	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6873	AssertRCReturn(rc, rc);
6874	Assert(pVmxTransient->cbInstr == 2);
6875
6876	pMixedCtx->rip += pVmxTransient->cbInstr;
6877	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6878
6879	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6880	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6881	pVmxTransient->fUpdateTscOffsetting = true;
6882	}
6883	else
6884	{
6885	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6886	rc = VERR_EM_INTERPRETER;
6887	}
6888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6889	return rc;
6890	}
6891
6892
6893	/**
6894	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6895	*/
6896	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6897	{
6898	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6899	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6900	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6901	AssertRCReturn(rc, rc);
6902
6903	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6904	if (RT_LIKELY(rc == VINF_SUCCESS))
6905	{
6906	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6907	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6908	AssertRCReturn(rc, rc);
6909	Assert(pVmxTransient->cbInstr == 3);
6910
6911	pMixedCtx->rip += pVmxTransient->cbInstr;
6912	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6913
6914	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6915	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6916	pVmxTransient->fUpdateTscOffsetting = true;
6917	}
6918	else
6919	{
6920	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6921	rc = VERR_EM_INTERPRETER;
6922	}
6923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6924	return rc;
6925	}
6926
6927
6928	/**
6929	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6930	*/
6931	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6932	{
6933	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6934	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6935	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6936	AssertRCReturn(rc, rc);
6937
6938	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6939	if (RT_LIKELY(rc == VINF_SUCCESS))
6940	{
6941	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6942	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6943	AssertRCReturn(rc, rc);
6944	Assert(pVmxTransient->cbInstr == 2);
6945
6946	pMixedCtx->rip += pVmxTransient->cbInstr;
6947	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6948	}
6949	else
6950	{
6951	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6952	rc = VERR_EM_INTERPRETER;
6953	}
6954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6955	return rc;
6956	}
6957
6958
6959	/**
6960	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6961	*/
6962	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6963	{
6964	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6965	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6966	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6967	AssertRCReturn(rc, rc);
6968
6969	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6970	rc = VBOXSTRICTRC_VAL(rc2);
6971	if (RT_LIKELY(rc == VINF_SUCCESS))
6972	{
6973	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6974	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6975	AssertRCReturn(rc, rc);
6976
6977	pMixedCtx->rip += pVmxTransient->cbInstr;
6978	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6979	}
6980	else
6981	{
6982	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6983	pVmxTransient->uExitQualification, rc));
6984	rc = VERR_EM_INTERPRETER;
6985	}
6986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6987	return rc;
6988	}
6989
6990
6991	/**
6992	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6993	*/
6994	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6995	{
6996	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6997	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6998	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6999	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7000	AssertRCReturn(rc, rc);
7001
7002	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7003	if (RT_LIKELY(rc == VINF_SUCCESS))
7004	{
7005	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7006	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7007	AssertRCReturn(rc, rc);
7008
7009	pMixedCtx->rip += pVmxTransient->cbInstr;
7010	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7011	}
7012	else
7013	{
7014	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7015	rc = VERR_EM_INTERPRETER;
7016	}
7017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7018	return rc;
7019	}
7020
7021
7022	/**
7023	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7024	*/
7025	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7026	{
7027	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7028	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7029	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7030	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7031	AssertRCReturn(rc, rc);
7032
7033	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7034	rc = VBOXSTRICTRC_VAL(rc2);
7035	if (RT_LIKELY( rc == VINF_SUCCESS
7036	\|\| rc == VINF_EM_HALT))
7037	{
7038	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7039	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7040	AssertRCReturn(rc3, rc3);
7041
7042	pMixedCtx->rip += pVmxTransient->cbInstr;
7043	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7044
7045	if ( rc == VINF_EM_HALT
7046	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7047	{
7048	rc = VINF_SUCCESS;
7049	}
7050	}
7051	else
7052	{
7053	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7054	rc = VERR_EM_INTERPRETER;
7055	}
7056	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7057	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7058	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7059	return rc;
7060	}
7061
7062
7063	/**
7064	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7065	*/
7066	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7067	{
7068	/*
7069	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7070	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7071	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7072	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7073	*/
7074	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7075	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7076	}
7077
7078
7079	/**
7080	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7081	*/
7082	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7083	{
7084	/*
7085	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7086	* root operation. If we get there there is something funny going on.
7087	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7088	*/
7089	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7090	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7091	}
7092
7093
7094	/**
7095	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7096	*/
7097	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7098	{
7099	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7100	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7101	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7102	}
7103
7104
7105	/**
7106	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7107	*/
7108	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7109	{
7110	/*
7111	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7112	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7113	* See Intel spec. 25.3 "Other Causes of VM-exits".
7114	*/
7115	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7116	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7117	}
7118
7119
7120	/**
7121	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7122	* VM-exit.
7123	*/
7124	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7125	{
7126	/*
7127	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7128	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7129	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7130	*/
7131	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7132	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7133	}
7134
7135
7136	/**
7137	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7138	* VM-exit.
7139	*/
7140	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7141	{
7142	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7143	return VINF_EM_RESET;
7144	}
7145
7146
7147	/**
7148	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7149	*/
7150	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7151	{
7152	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7153	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7154	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7155	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7156	AssertRCReturn(rc, rc);
7157
7158	pMixedCtx->rip++;
7159	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7160	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7161	rc = VINF_SUCCESS;
7162	else
7163	rc = VINF_EM_HALT;
7164
7165	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7166	return rc;
7167	}
7168
7169
7170	/**
7171	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7172	*/
7173	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7174	{
7175	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7176	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7177	AssertRCReturn(rc, rc);
7178	return rc;
7179	}
7180
7181
7182	/**
7183	* VM-exit handler for expiry of the VMX preemption timer.
7184	*/
7185	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7186	{
7187	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7188	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7189	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7190	int rc = fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7191	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7192	return rc;
7193	}
7194
7195
7196	/**
7197	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7198	*/
7199	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7200	{
7201	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7202	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7203	/** @todo check if XSETBV is supported by the recompiler. */
7204	return VERR_EM_INTERPRETER;
7205	}
7206
7207
7208	/**
7209	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7210	*/
7211	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7212	{
7213	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7214	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7215	/** @todo implement EMInterpretInvpcid() */
7216	return VERR_EM_INTERPRETER;
7217	}
7218
7219
7220	/**
7221	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7222	* Error VM-exit.
7223	*/
7224	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225	{
7226	uint32_t uIntrState;
7227	RTHCUINTREG uHCReg;
7228	uint64_t u64Val;
7229
7230	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7231	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7232	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7233	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7234	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7235	AssertRCReturn(rc, rc);
7236
7237	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7238	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7239	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7240	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7241
7242	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7243	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7244	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7245	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7246	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7247	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7248	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7249	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7250	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7251	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7252	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7253	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7254
7255	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7256
7257	return VERR_VMX_INVALID_GUEST_STATE;
7258	}
7259
7260
7261	/**
7262	* VM-exit handler for VM-entry failure due to an MSR-load
7263	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7264	*/
7265	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7266	{
7267	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7268	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7269	}
7270
7271
7272	/**
7273	* VM-exit handler for VM-entry failure due to a machine-check event
7274	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7275	*/
7276	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7277	{
7278	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7279	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7280	}
7281
7282
7283	/**
7284	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7285	* theory.
7286	*/
7287	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7288	{
7289	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7290	pVM, pVCpu, pMixedCtx));
7291	return VERR_VMX_UNDEFINED_EXIT_CODE;
7292	}
7293
7294
7295	/**
7296	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7297	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7298	* Conditional VM-exit.
7299	*/
7300	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7301	{
7302	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7303	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7304	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7305	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7306	return VERR_EM_INTERPRETER;
7307	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7308	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7309	}
7310
7311
7312	/**
7313	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7314	*/
7315	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7316	{
7317	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7318	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7319	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7320	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7321	return VERR_EM_INTERPRETER;
7322	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7323	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7324	}
7325
7326
7327	/**
7328	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7329	*/
7330	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7331	{
7332	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7333	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7334	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7335	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7336	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7337	AssertRCReturn(rc, rc);
7338
7339	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7340	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7341	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7343
7344	/* Update RIP and continue guest execution. */
7345	if (RT_LIKELY(rc == VINF_SUCCESS))
7346	{
7347	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7348	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7349	AssertRCReturn(rc, rc);
7350
7351	Assert(pVmxTransient->cbInstr == 2);
7352	pMixedCtx->rip += pVmxTransient->cbInstr;
7353	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7354	}
7355	return rc;
7356	}
7357
7358
7359	/**
7360	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7361	*/
7362	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7363	{
7364	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7365	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7366	AssertRCReturn(rc, rc);
7367	Assert(pVmxTransient->cbInstr == 2);
7368
7369	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7370	if ( pVM->hm.s.fTPRPatchingActive
7371	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7372	{
7373	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7374	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7375	{
7376	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7377	AssertRC(rc);
7378	}
7379
7380	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7381	AssertRCReturn(rc, rc);
7382	pMixedCtx->rip += pVmxTransient->cbInstr;
7383	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7384	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7385	return VINF_SUCCESS;
7386	}
7387
7388	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7389	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7390	{
7391	switch (pMixedCtx->ecx)
7392	{
7393	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7394	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7395	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7396	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7397	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7398	}
7399	}
7400	#ifdef DEBUG
7401	else
7402	{
7403	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7404	switch (pMixedCtx->ecx)
7405	{
7406	case MSR_IA32_SYSENTER_CS:
7407	case MSR_IA32_SYSENTER_EIP:
7408	case MSR_IA32_SYSENTER_ESP:
7409	case MSR_K8_FS_BASE:
7410	case MSR_K8_GS_BASE:
7411	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7412	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7413	case MSR_K8_LSTAR:
7414	case MSR_K6_STAR:
7415	case MSR_K8_SF_MASK:
7416	case MSR_K8_TSC_AUX:
7417	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7418	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7419	}
7420	}
7421	#endif
7422
7423	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7424	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7425	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7426	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7427	AssertRCReturn(rc, rc);
7428
7429	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7430	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7432
7433	/* Update guest-state and continue execution. */
7434	if (RT_LIKELY(rc == VINF_SUCCESS))
7435	{
7436	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7437	AssertRCReturn(rc, rc);
7438
7439	pMixedCtx->rip += pVmxTransient->cbInstr;
7440	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7441
7442	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7443	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7444	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7445	{
7446	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7447	}
7448	}
7449	return rc;
7450	}
7451
7452
7453	/**
7454	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7455	*/
7456	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7457	{
7458	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7459	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7461	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7462	return VERR_EM_INTERPRETER;
7463	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7464	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7465	}
7466
7467
7468	/**
7469	* VM-exit handler for when the TPR value is lowered below the specified
7470	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7471	*/
7472	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7473	{
7474	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7475	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7476
7477	/*
7478	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7479	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7480	* resume guest execution.
7481	*/
7482	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7483	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7484	return VINF_SUCCESS;
7485	}
7486
7487
7488	/**
7489	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7490	* VM-exit.
7491	*
7492	* @retval VINF_SUCCESS when guest execution can continue.
7493	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7494	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7495	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7496	* recompiler.
7497	*/
7498	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7499	{
7500	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7501	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7502	AssertRCReturn(rc, rc);
7503
7504	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7505	const uint8_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7506	switch (uAccessType)
7507	{
7508	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7509	{
7510	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7511	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7512	AssertRCReturn(rc, rc);
7513
7514	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7515	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7516	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7517	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7518
7519	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7520	{
7521	case 0: /* CR0 */
7522	Log(("CR0 write rc=%d\n", rc));
7523	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7524	break;
7525	case 2: /* CR2 */
7526	Log(("CR2 write rc=%d\n", rc));
7527	break;
7528	case 3: /* CR3 */
7529	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7530	Log(("CR3 write rc=%d\n", rc));
7531	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7532	break;
7533	case 4: /* CR4 */
7534	Log(("CR4 write rc=%d\n", rc));
7535	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7536	break;
7537	case 8: /* CR8 */
7538	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7539	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7540	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7541	break;
7542	default:
7543	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7544	break;
7545	}
7546
7547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7548	break;
7549	}
7550
7551	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7552	{
7553	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7554	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7555	AssertRCReturn(rc, rc);
7556	Assert( !pVM->hm.s.fNestedPaging
7557	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7558	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7559
7560	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7561	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7562	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7563
7564	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7565	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7566	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7567	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7568	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7569	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7570	break;
7571	}
7572
7573	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7574	{
7575	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7576	rc \|= EMInterpretCLTS(pVM, pVCpu);
7577	AssertRCReturn(rc, rc);
7578	if (RT_LIKELY(rc == VINF_SUCCESS))
7579	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7581	Log(("CRX CLTS write rc=%d\n", rc));
7582	break;
7583	}
7584
7585	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7586	{
7587	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7588	rc \|= EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7589	AssertRCReturn(rc, rc);
7590	if (RT_LIKELY(rc == VINF_SUCCESS))
7591	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7593	Log(("CRX LMSW write rc=%d\n", rc));
7594	break;
7595	}
7596
7597	default:
7598	{
7599	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7600	return VERR_VMX_UNEXPECTED_EXCEPTION;
7601	}
7602	}
7603
7604	/* Validate possible error codes. */
7605	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7606	if (RT_SUCCESS(rc))
7607	{
7608	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7609	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7610	AssertRCReturn(rc2, rc2);
7611	pMixedCtx->rip += pVmxTransient->cbInstr;
7612	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7613	}
7614
7615	return rc;
7616	}
7617
7618
7619	/**
7620	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7621	* VM-exit.
7622	*/
7623	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7624	{
7625	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7626
7627	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7628	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7629	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7630	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7631	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7632	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7633	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7634	AssertRCReturn(rc, rc);
7635
7636	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7637	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7638	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7639	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7640	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7641	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7642	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7643
7644	/* I/O operation lookup arrays. */
7645	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7646	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7647
7648	const uint32_t cbSize = s_aIOSize[uIOWidth];
7649	const uint32_t cbInstr = pVmxTransient->cbInstr;
7650	if (fIOString)
7651	{
7652	/* INS/OUTS - I/O String instruction. */
7653	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7654	/** @todo for now manually disassemble later optimize by getting the fields from
7655	* the VMCS. */
7656	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7657	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7658	* segment prefix info. */
7659	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7660	if (RT_SUCCESS(rc))
7661	{
7662	if (fIOWrite)
7663	{
7664	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7665	(DISCPUMODE)pDis->uAddrMode, cbSize);
7666	rc = VBOXSTRICTRC_VAL(rc2);
7667	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7668	}
7669	else
7670	{
7671	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7672	(DISCPUMODE)pDis->uAddrMode, cbSize);
7673	rc = VBOXSTRICTRC_VAL(rc2);
7674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7675	}
7676	}
7677	else
7678	{
7679	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7680	rc = VINF_EM_RAW_EMULATE_INSTR;
7681	}
7682	}
7683	else
7684	{
7685	/* IN/OUT - I/O instruction. */
7686	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7687	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7688	if (fIOWrite)
7689	{
7690	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7691	rc = VBOXSTRICTRC_VAL(rc2);
7692	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7693	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7695	}
7696	else
7697	{
7698	uint32_t u32Result = 0;
7699	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7700	rc = VBOXSTRICTRC_VAL(rc2);
7701	if (IOM_SUCCESS(rc))
7702	{
7703	/* Save result of I/O IN instr. in AL/AX/EAX. */
7704	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7705	}
7706	else if (rc == VINF_IOM_R3_IOPORT_READ)
7707	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7709	}
7710	}
7711
7712	if (IOM_SUCCESS(rc))
7713	{
7714	pMixedCtx->rip += cbInstr;
7715	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7716	if (RT_LIKELY(rc == VINF_SUCCESS))
7717	{
7718	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7719	AssertRCReturn(rc, rc);
7720
7721	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7722	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7723	{
7724	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7725	for (unsigned i = 0; i < 4; i++)
7726	{
7727	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7728	if ( ( uIOPort >= pMixedCtx->dr[i]
7729	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7730	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7731	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7732	{
7733	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7734	uint64_t uDR6 = ASMGetDR6();
7735
7736	/* Clear all breakpoint status flags and set the one we just hit. */
7737	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7738	uDR6 \|= (uint64_t)RT_BIT(i);
7739
7740	/*
7741	* Note: AMD64 Architecture Programmer's Manual 13.1:
7742	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7743	* be cleared by software after the contents have been read.
7744	*/
7745	ASMSetDR6(uDR6);
7746
7747	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7748	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7749
7750	/* Paranoia. */
7751	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7752	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7753	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7754
7755	/* Resync DR7 */
7756	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7757	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7758
7759	/* Inject #DB and get on with guest execution. */
7760	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7761	AssertRCReturn(rc, rc);
7762	break;
7763	}
7764	}
7765	}
7766	}
7767	}
7768
7769	#ifdef DEBUG
7770	if (rc == VINF_IOM_R3_IOPORT_READ)
7771	Assert(!fIOWrite);
7772	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7773	Assert(fIOWrite);
7774	else
7775	{
7776	AssertMsg( RT_FAILURE(rc)
7777	\|\| rc == VINF_SUCCESS
7778	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7779	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7780	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7781	}
7782	#endif
7783
7784	return rc;
7785	}
7786
7787
7788	/**
7789	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7790	* VM-exit.
7791	*/
7792	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7793	{
7794	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7795
7796	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7797	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7798	AssertRCReturn(rc, rc);
7799	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7800	{
7801	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7802	AssertRCReturn(rc, rc);
7803	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7804	{
7805	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7806	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7807	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7808	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7809	{
7810	/* Save it as a pending event while will be converted to a TRPM event on the way out to ring-3. */
7811	pVCpu->hm.s.Event.fPending = true;
7812	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7813	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7814	AssertRCReturn(rc, rc);
7815	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7816	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7817	else
7818	pVCpu->hm.s.Event.u32ErrCode = 0;
7819	}
7820	}
7821	}
7822	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7823	* emulation. */
7824	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7825	return VERR_EM_INTERPRETER;
7826	}
7827
7828
7829	/**
7830	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7831	*/
7832	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7833	{
7834	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7835	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7836	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7837	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7838	AssertRCReturn(rc, rc);
7839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7840	return VINF_EM_DBG_STOP;
7841	}
7842
7843
7844	/**
7845	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7846	*/
7847	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7848	{
7849	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7850	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7851
7852	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7853	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7854	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7855	return VINF_SUCCESS;
7856	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7857	return rc;
7858
7859	#if 0
7860	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7861	* just sync the whole thing. */
7862	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7863	#else
7864	/* Aggressive state sync. for now. */
7865	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7866	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7867	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7868	#endif
7869	AssertRCReturn(rc, rc);
7870
7871	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7872	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7873	switch (uAccessType)
7874	{
7875	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7876	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7877	{
7878	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7879	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7880	{
7881	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7882	}
7883
7884	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7885	GCPhys &= PAGE_BASE_GC_MASK;
7886	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7887	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7888	CPUMCTX2CORE(pMixedCtx), GCPhys);
7889	rc = VBOXSTRICTRC_VAL(rc2);
7890	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7891	if ( rc == VINF_SUCCESS
7892	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7893	\|\| rc == VERR_PAGE_NOT_PRESENT)
7894	{
7895	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7896	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7897	rc = VINF_SUCCESS;
7898	}
7899	break;
7900	}
7901
7902	default:
7903	rc = VINF_EM_RAW_EMULATE_INSTR;
7904	break;
7905	}
7906
7907	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7908	return rc;
7909	}
7910
7911
7912	/**
7913	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7914	* VM-exit.
7915	*/
7916	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7917	{
7918	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7919
7920	/* We should -not- get this VM-exit if the guest is debugging. */
7921	if (CPUMIsGuestDebugStateActive(pVCpu))
7922	{
7923	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7924	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7925	}
7926
7927	int rc = VERR_INTERNAL_ERROR_5;
7928	if ( !DBGFIsStepping(pVCpu)
7929	&& !CPUMIsHyperDebugStateActive(pVCpu))
7930	{
7931	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7932
7933	/* Don't intercept MOV DRx. */
7934	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7935	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7936	AssertRCReturn(rc, rc);
7937
7938	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7939	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7940	AssertRCReturn(rc, rc);
7941	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7942
7943	#ifdef VBOX_WITH_STATISTICS
7944	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7945	AssertRCReturn(rc, rc);
7946	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7948	else
7949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7950	#endif
7951	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7952	return VINF_SUCCESS;
7953	}
7954
7955	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7956	* time and restore DRx registers afterwards */
7957	/*
7958	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7959	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7960	*/
7961	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7962	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7963	AssertRCReturn(rc, rc);
7964
7965	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7966	{
7967	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7968	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7969	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7970	if (RT_SUCCESS(rc))
7971	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7973	}
7974	else
7975	{
7976	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7977	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7978	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7980	}
7981
7982	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7983	if (RT_SUCCESS(rc))
7984	{
7985	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7986	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7987	AssertRCReturn(rc2, rc2);
7988	pMixedCtx->rip += pVmxTransient->cbInstr;
7989	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7990	}
7991	return rc;
7992	}
7993
7994
7995	/**
7996	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7997	* Conditional VM-exit.
7998	*/
7999	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8000	{
8001	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8002	Assert(pVM->hm.s.fNestedPaging);
8003
8004	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8005	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8006	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8007	return VINF_SUCCESS;
8008	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8009	return rc;
8010
8011	RTGCPHYS GCPhys = 0;
8012	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8013	AssertRCReturn(rc, rc);
8014
8015	#if 0
8016	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8017	#else
8018	/* Aggressive state sync. for now. */
8019	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8020	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8021	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8022	#endif
8023	AssertRCReturn(rc, rc);
8024
8025	/*
8026	* If we succeed, resume guest execution.
8027	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8028	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8029	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8030	* weird case. See @bugref{6043}.
8031	*/
8032	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8033	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8034	rc = VBOXSTRICTRC_VAL(rc2);
8035	if ( rc == VINF_SUCCESS
8036	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8037	\|\| rc == VERR_PAGE_NOT_PRESENT)
8038	{
8039	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8040	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8041	return VINF_SUCCESS;
8042	}
8043	return rc;
8044	}
8045
8046
8047	/**
8048	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8049	* VM-exit.
8050	*/
8051	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8052	{
8053	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8054	Assert(pVM->hm.s.fNestedPaging);
8055
8056	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8057	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8058	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8059	return VINF_SUCCESS;
8060	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8061	return rc;
8062
8063	RTGCPHYS GCPhys = 0;
8064	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8065	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8066	#if 0
8067	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8068	#else
8069	/* Aggressive state sync. for now. */
8070	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8071	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8072	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8073	#endif
8074	AssertRCReturn(rc, rc);
8075
8076	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8077	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8078
8079	RTGCUINT uErrorCode = 0;
8080	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8081	uErrorCode \|= X86_TRAP_PF_ID;
8082	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8083	uErrorCode \|= X86_TRAP_PF_RW;
8084	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8085	uErrorCode \|= X86_TRAP_PF_P;
8086
8087	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8088	TRPMSetErrorCode(pVCpu, uErrorCode);
8089	TRPMSetFaultAddress(pVCpu, GCPhys);
8090
8091	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8092	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8093
8094	/* Handle the pagefault trap for the nested shadow table. */
8095	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8096	TRPMResetTrap(pVCpu);
8097
8098	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8099	if ( rc == VINF_SUCCESS
8100	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8101	\|\| rc == VERR_PAGE_NOT_PRESENT)
8102	{
8103	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8104	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8105	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8106	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8107	return VINF_SUCCESS;
8108	}
8109
8110	Log(("EPT return to ring-3 rc=%d\n"));
8111
8112	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8113	/** @todo Shouldn't we update TRPM here? */
8114	return rc;
8115	}
8116
8117
8118	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8119	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8120	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8121	/**
8122	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8123	*/
8124	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8125	{
8126	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8127
8128	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8129	AssertRCReturn(rc, rc);
8130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8131
8132	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8133	{
8134	/* Old-style FPU error reporting needs some extra work. */
8135	/** @todo don't fall back to the recompiler, but do it manually. */
8136	return VERR_EM_INTERPRETER;
8137	}
8138	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8139	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8140	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8141	AssertRCReturn(rc, rc);
8142	return rc;
8143	}
8144
8145
8146	/**
8147	* VM-exit exception handler for #BP (Breakpoint exception).
8148	*/
8149	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8150	{
8151	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8152
8153	/** @todo Try optimize this by not saving the entire guest state unless
8154	* really needed. */
8155	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8156	AssertRCReturn(rc, rc);
8157	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8158
8159	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8160	if (rc == VINF_EM_RAW_GUEST_TRAP)
8161	{
8162	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8163	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8164	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8165	AssertRCReturn(rc, rc);
8166
8167	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8168	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8169	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8170	AssertRCReturn(rc, rc);
8171	}
8172
8173	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8174	return rc;
8175	}
8176
8177
8178	/**
8179	* VM-exit exception handler for #DB (Debug exception).
8180	*/
8181	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8182	{
8183	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8184
8185	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8186	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8187	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8188	AssertRCReturn(rc, rc);
8189
8190	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8191	uint64_t uDR6 = X86_DR6_INIT_VAL;
8192	uDR6 \|= (pVmxTransient->uExitQualification
8193	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8194	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8195	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8196	if (rc == VINF_EM_RAW_GUEST_TRAP)
8197	{
8198	/** @todo revisit this. */
8199	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8200	pMixedCtx->dr[6] = uDR6;
8201
8202	if (CPUMIsGuestDebugStateActive(pVCpu))
8203	ASMSetDR6(pMixedCtx->dr[6]);
8204
8205	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8206	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8207
8208	/* Paranoia. */
8209	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8210	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8211	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8212
8213	/* Resync DR7. */
8214	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8215
8216	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8217	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8218	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8219	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8220	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8221	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8222	AssertRCReturn(rc,rc);
8223	return rc;
8224	}
8225	/* Return to ring 3 to deal with the debug exit code. */
8226	return rc;
8227	}
8228
8229
8230	/**
8231	* VM-exit exception handler for #NM (Device-not-available exception: floating
8232	* point exception).
8233	*/
8234	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8235	{
8236	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8237
8238	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8239	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8240	#endif
8241
8242	/* We require CR0 and EFER. EFER is always up-to-date. */
8243	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8244	AssertRCReturn(rc, rc);
8245
8246	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8247	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8248	if (rc == VINF_SUCCESS)
8249	{
8250	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8251	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8252	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8253	return VINF_SUCCESS;
8254	}
8255
8256	/* Forward #NM to the guest. */
8257	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8258	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8259	AssertRCReturn(rc, rc);
8260	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8261	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8262	pVmxTransient->cbInstr, 0 /* error code */);
8263	AssertRCReturn(rc, rc);
8264	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8265	return rc;
8266	}
8267
8268
8269	/**
8270	* VM-exit exception handler for #GP (General-protection exception).
8271	*
8272	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8273	*/
8274	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8275	{
8276	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8277
8278	int rc = VERR_INTERNAL_ERROR_5;
8279
8280	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8281	{
8282	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8283	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8284	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8285	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8286	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8287	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8288	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8289	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8290	AssertRCReturn(rc, rc);
8291	return rc;
8292	#else
8293	/* We don't intercept #GP. */
8294	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8295	return VERR_VMX_UNEXPECTED_EXCEPTION;
8296	#endif
8297	}
8298
8299	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8300	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8301
8302	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8303	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8304	AssertRCReturn(rc, rc);
8305
8306	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8307	unsigned int cbOp = 0;
8308	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8309	if (RT_SUCCESS(rc))
8310	{
8311	rc = VINF_SUCCESS;
8312	Assert(cbOp == pDis->cbInstr);
8313	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8314	switch (pDis->pCurInstr->uOpcode)
8315	{
8316	case OP_CLI:
8317	pMixedCtx->eflags.Bits.u1IF = 0;
8318	pMixedCtx->rip += pDis->cbInstr;
8319	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8320	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8321	break;
8322
8323	case OP_STI:
8324	pMixedCtx->eflags.Bits.u1IF = 1;
8325	pMixedCtx->rip += pDis->cbInstr;
8326	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8327	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8328	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE \| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8329	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8330	break;
8331
8332	case OP_HLT:
8333	rc = VINF_EM_HALT;
8334	pMixedCtx->rip += pDis->cbInstr;
8335	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8337	break;
8338
8339	case OP_POPF:
8340	{
8341	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8342	size_t cbParm = 0;
8343	uint32_t uMask = 0;
8344	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8345	{
8346	cbParm = 4;
8347	uMask = 0xffffffff;
8348	}
8349	else
8350	{
8351	cbParm = 2;
8352	uMask = 0xffff;
8353	}
8354
8355	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8356	RTGCPTR GCPtrStack = 0;
8357	X86EFLAGS uEflags;
8358	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8359	&GCPtrStack);
8360	if (RT_SUCCESS(rc))
8361	{
8362	Assert(sizeof(uEflags.u32) >= cbParm);
8363	uEflags.u32 = 0;
8364	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8365	}
8366	if (RT_FAILURE(rc))
8367	{
8368	rc = VERR_EM_INTERPRETER;
8369	break;
8370	}
8371	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8372	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8373	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8374	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8375	pMixedCtx->eflags.Bits.u1RF = 0;
8376	pMixedCtx->esp += cbParm;
8377	pMixedCtx->esp &= uMask;
8378	pMixedCtx->rip += pDis->cbInstr;
8379	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8380	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8381	break;
8382	}
8383
8384	case OP_PUSHF:
8385	{
8386	size_t cbParm = 0;
8387	uint32_t uMask = 0;
8388	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8389	{
8390	cbParm = 4;
8391	uMask = 0xffffffff;
8392	}
8393	else
8394	{
8395	cbParm = 2;
8396	uMask = 0xffff;
8397	}
8398
8399	/* Get the stack pointer & push the contents of eflags onto the stack. */
8400	RTGCPTR GCPtrStack = 0;
8401	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8402	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8403	if (RT_FAILURE(rc))
8404	{
8405	rc = VERR_EM_INTERPRETER;
8406	break;
8407	}
8408	X86EFLAGS uEflags;
8409	uEflags = pMixedCtx->eflags;
8410	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8411	uEflags.Bits.u1RF = 0;
8412	uEflags.Bits.u1VM = 0;
8413
8414	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8415	if (RT_FAILURE(rc))
8416	{
8417	rc = VERR_EM_INTERPRETER;
8418	break;
8419	}
8420	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8421	pMixedCtx->esp -= cbParm;
8422	pMixedCtx->esp &= uMask;
8423	pMixedCtx->rip += pDis->cbInstr;
8424	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8426	break;
8427	}
8428
8429	case OP_IRET:
8430	{
8431	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8432	* instruction reference. */
8433	RTGCPTR GCPtrStack = 0;
8434	uint32_t uMask = 0xffff;
8435	uint16_t aIretFrame[3];
8436	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8437	{
8438	rc = VERR_EM_INTERPRETER;
8439	break;
8440	}
8441	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8442	&GCPtrStack);
8443	if (RT_SUCCESS(rc))
8444	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8445	if (RT_FAILURE(rc))
8446	{
8447	rc = VERR_EM_INTERPRETER;
8448	break;
8449	}
8450	pMixedCtx->eip = 0;
8451	pMixedCtx->ip = aIretFrame[0];
8452	pMixedCtx->cs.Sel = aIretFrame[1];
8453	pMixedCtx->cs.ValidSel = aIretFrame[1];
8454	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8455	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8456	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8457	pMixedCtx->sp += sizeof(aIretFrame);
8458	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8459	\| HM_CHANGED_GUEST_RFLAGS;
8460	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8462	break;
8463	}
8464
8465	case OP_INT:
8466	{
8467	uint16_t uVector = pDis->Param1.uValue & 0xff;
8468	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8469	AssertRCReturn(rc, rc);
8470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8471	break;
8472	}
8473
8474	case OP_INTO:
8475	{
8476	if (pMixedCtx->eflags.Bits.u1OF)
8477	{
8478	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8480	}
8481	break;
8482	}
8483
8484	default:
8485	{
8486	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8487	EMCODETYPE_SUPERVISOR);
8488	rc = VBOXSTRICTRC_VAL(rc2);
8489	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8490	Log2(("#GP rc=%Rrc\n", rc));
8491	break;
8492	}
8493	}
8494	}
8495	else
8496	rc = VERR_EM_INTERPRETER;
8497
8498	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8499	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8500	("#GP Unexpected rc=%Rrc\n", rc));
8501	return rc;
8502	}
8503
8504
8505	/**
8506	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8507	* the exception reported in the VMX transient structure back into the VM.
8508	*
8509	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8510	* VMX transient structure to be up-to-date.
8511	*/
8512	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8513	{
8514	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8515
8516	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8517	hmR0VmxCheckExitDueToEventDelivery(). */
8518	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8519	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8520	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8521	AssertRCReturn(rc, rc);
8522	return rc;
8523	}
8524
8525
8526	/**
8527	* VM-exit exception handler for #PF (Page-fault exception).
8528	*/
8529	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8530	{
8531	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8532
8533	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8534	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8535	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8536	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8537	AssertRCReturn(rc, rc);
8538
8539	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8540	if (pVM->hm.s.fNestedPaging)
8541	{
8542	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8543	{
8544	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8545	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8546	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8547	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8548	AssertRCReturn(rc, rc);
8549	}
8550	else
8551	{
8552	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8553	Assert(!pVCpu->hm.s.Event.fPending);
8554	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8555	}
8556	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8557	return rc;
8558	}
8559	#else
8560	Assert(!pVM->hm.s.fNestedPaging);
8561	#endif
8562
8563	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8564	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8565	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8566	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8567	AssertRCReturn(rc, rc);
8568	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8569	if ( pVM->hm.s.fTRPPatchingAllowed
8570	&& pVM->hm.s.pGuestPatchMem
8571	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8572	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8573	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8574	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8575	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8576	{
8577	RTGCPHYS GCPhys;
8578	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8579	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8580	if ( rc == VINF_SUCCESS
8581	&& GCPhys == GCPhysApicBase)
8582	{
8583	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8584	AssertRCReturn(rc, rc);
8585
8586	/* Only attempt to patch the instruction once. */
8587	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8588	if (!pPatch)
8589	return VINF_EM_HM_PATCH_TPR_INSTR;
8590	}
8591	}
8592	#endif
8593
8594	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8595	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8596	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8597
8598	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8599	AssertRCReturn(rc, rc);
8600
8601	/* Forward it to the trap handler first. */
8602	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8603	(RTGCPTR)pVmxTransient->uExitQualification);
8604
8605	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8606	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8607
8608	if (rc == VINF_SUCCESS)
8609	{
8610	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8611	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8612	* memory? We don't update the whole state here... */
8613	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8614	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8615
8616	TRPMResetTrap(pVCpu);
8617	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8618	return rc;
8619	}
8620	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8621	{
8622	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8623	{
8624	/* It's a guest page fault and needs to be reflected to the guest. */
8625	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8626	TRPMResetTrap(pVCpu);
8627	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8628	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8629	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8630	pVmxTransient->cbInstr, uGstErrorCode);
8631	AssertRCReturn(rc, rc);
8632	}
8633	else
8634	{
8635	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8636	Assert(!pVCpu->hm.s.Event.fPending);
8637	TRPMResetTrap(pVCpu);
8638	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8639	}
8640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8641	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8642	return rc;
8643	}
8644
8645	TRPMResetTrap(pVCpu);
8646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8647	return rc;
8648	}
8649

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45379

Download in other formats: