HMVMXR0.cpp@ 45387

Last change on this file since 45387 was 45387, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 361.1 KB

Line
1	/* $Id: HMVMXR0.cpp 45387 2013-04-05 21:37:40Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
76	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(19)
77	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(21)
79	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
80	\| VMX_UPDATED_GUEST_RIP \
81	\| VMX_UPDATED_GUEST_RSP \
82	\| VMX_UPDATED_GUEST_RFLAGS \
83	\| VMX_UPDATED_GUEST_CR0 \
84	\| VMX_UPDATED_GUEST_CR3 \
85	\| VMX_UPDATED_GUEST_CR4 \
86	\| VMX_UPDATED_GUEST_GDTR \
87	\| VMX_UPDATED_GUEST_IDTR \
88	\| VMX_UPDATED_GUEST_LDTR \
89	\| VMX_UPDATED_GUEST_TR \
90	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
91	\| VMX_UPDATED_GUEST_DEBUG \
92	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
96	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
97	\| VMX_UPDATED_GUEST_INTR_STATE \
98	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
99	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
100	\| VMX_UPDATED_GUEST_APIC_STATE)
101
102	/**
103	* Flags to skip redundant reads of some common VMCS fields.
104	*/
105	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
106	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
107	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
108	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
109	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
110	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
111
112	/*
113	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
114	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
115	* we have Nested Paging support.
116	*/
117	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/* Maximum VM-instruction error number. */
126	#define VMX_INSTR_ERROR_MAX 28
127
128	/*******************************************************************************
129	* Structures and Typedefs *
130	*******************************************************************************/
131	/**
132	* A state structure for holding miscellaneous information across
133	* VMX non-root operation and restored after the transition.
134	*/
135	typedef struct VMXTRANSIENT
136	{
137	/** The host's rflags/eflags. */
138	RTCCUINTREG uEFlags;
139	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
140	uint64_t u64LStarMsr;
141	/** The guest's TPR value used for TPR shadowing. */
142	uint8_t u8GuestTpr;
143
144	/** The basic VM-exit reason. */
145	uint16_t uExitReason;
146	/** The VM-exit exit qualification. */
147	RTGCUINTPTR uExitQualification;
148	/** The VM-exit interruption error code. */
149	uint32_t uExitIntrErrorCode;
150
151	/** The VM-exit interruption-information field. */
152	uint32_t uExitIntrInfo;
153	/** Whether the VM-entry failed or not. */
154	bool fVMEntryFailed;
155	/** The VM-exit instruction-length field. */
156	uint32_t cbInstr;
157
158	/** The VM-entry interruption-information field. */
159	uint32_t uEntryIntrInfo;
160	/** The VM-entry exception error code field. */
161	uint32_t uEntryXcptErrorCode;
162	/** The VM-entry instruction length field. */
163	uint32_t cbEntryInstr;
164
165	/** IDT-vectoring information field. */
166	uint32_t uIdtVectoringInfo;
167	/** IDT-vectoring error code. */
168	uint32_t uIdtVectoringErrorCode;
169
170	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
171	uint32_t fVmcsFieldsRead;
172	/** Whether TSC-offsetting should be setup before VM-entry. */
173	bool fUpdateTscOffsetting;
174	/** Whether the VM-exit was caused by a page-fault during delivery of a
175	* contributary exception or a page-fault. */
176	bool fVectoringPF;
177	} VMXTRANSIENT, *PVMXTRANSIENT;
178
179	/**
180	* MSR-bitmap read permissions.
181	*/
182	typedef enum VMXMSREXITREAD
183	{
184	/** Reading this MSR causes a VM-exit. */
185	VMXMSREXIT_INTERCEPT_READ = 0xb,
186	/** Reading this MSR does not cause a VM-exit. */
187	VMXMSREXIT_PASSTHRU_READ
188	} VMXMSREXITREAD;
189
190	/**
191	* MSR-bitmap write permissions.
192	*/
193	typedef enum VMXMSREXITWRITE
194	{
195	/** Writing to this MSR causes a VM-exit. */
196	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
197	/** Writing to this MSR does not cause a VM-exit. */
198	VMXMSREXIT_PASSTHRU_WRITE
199	} VMXMSREXITWRITE;
200
201	/*******************************************************************************
202	* Internal Functions *
203	*******************************************************************************/
204	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
205	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
206	uint32_t cbInstr, uint32_t u32ErrCode);
207
208	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259
260	/*******************************************************************************
261	* Global Variables *
262	*******************************************************************************/
263	/** @todo Move this to hm_vmx.h. */
264	/**
265	* VM-exit handler.
266	*
267	* @returns VBox status code.
268	* @param pVM Pointer to the VM.
269	* @param pVCpu Pointer to the VMCPU.
270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
271	* out-of-sync. Make sure to update the required
272	* fields before using them.
273	* @param pVmxTransient Pointer to the VMX-transient structure.
274	*/
275	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
278
279	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
280	{
281	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
282	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
283	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
284	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
285	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
286	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
287	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
288	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
289	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
290	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
291	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
292	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
293	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
294	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
295	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
296	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
297	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
298	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
299	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
300	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
301	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
302	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
303	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
304	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
305	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
306	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
307	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
308	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
309	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
310	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
311	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
312	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
313	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
314	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
315	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
316	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
317	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
318	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
319	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
320	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
321	/* 40 UNDEFINED */ hmR0VmxExitPause,
322	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
323	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
324	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
325	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
326	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
327	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
328	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
329	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
330	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
331	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
332	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
333	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
334	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
335	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
336	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
337	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
338	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
339	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
340	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
341	};
342
343	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
344	{
345	/* 0 */ "(Not Used)",
346	/* 1 */ "VMCALL executed in VMX root operation.",
347	/* 2 */ "VMCLEAR with invalid physical address.",
348	/* 3 */ "VMCLEAR with VMXON pointer.",
349	/* 4 */ "VMLAUNCH with non-clear VMCS.",
350	/* 5 */ "VMRESUME with non-launched VMCS.",
351	/* 6 */ "VMRESUME after VMXOFF",
352	/* 7 */ "VM entry with invalid control fields.",
353	/* 8 */ "VM entry with invalid host state fields.",
354	/* 9 */ "VMPTRLD with invalid physical address.",
355	/* 10 */ "VMPTRLD with VMXON pointer.",
356	/* 11 */ "VMPTRLD with incorrect revision identifier.",
357	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
358	/* 13 */ "VMWRITE to read-only VMCS component.",
359	/* 14 */ "(Not Used)",
360	/* 15 */ "VMXON executed in VMX root operation.",
361	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
362	/* 17 */ "VM entry with non-launched executing VMCS.",
363	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
364	/* 19 */ "VMCALL with non-clear VMCS.",
365	/* 20 */ "VMCALL with invalid VM-exit control fields.",
366	/* 21 */ "(Not Used)",
367	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
368	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
369	/* 24 */ "VMCALL with invalid SMM-monitor features.",
370	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
371	/* 26 */ "VM entry with events blocked by MOV SS.",
372	/* 27 */ "(Not Used)",
373	/* 28 */ "Invalid operand to INVEPT/INVVPID."
374	};
375
376
377	/**
378	* Updates the VM's last error record. If there was a VMX instruction error,
379	* reads the error data from the VMCS and updates VCPU's last error record as
380	* well.
381	*
382	* @param pVM Pointer to the VM.
383	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
384	* VERR_VMX_GENERIC).
385	* @param rc The error code.
386	*/
387	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
388	{
389	AssertPtr(pVM);
390	if (rc == VERR_VMX_GENERIC)
391	{
392	AssertPtrReturnVoid(pVCpu);
393	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
394	}
395	pVM->hm.s.lLastError = rc;
396	}
397
398
399	/**
400	* Reads the VM-entry interruption-information field from the VMCS into the VMX
401	* transient structure.
402	*
403	* @returns VBox status code.
404	* @param pVmxTransient Pointer to the VMX transient structure.
405	*
406	* @remarks No-long-jump zone!!!
407	*/
408	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
409	{
410	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
411	AssertRCReturn(rc, rc);
412	return VINF_SUCCESS;
413	}
414
415
416	/**
417	* Reads the VM-entry exception error code field from the VMCS into
418	* the VMX transient structure.
419	*
420	* @returns VBox status code.
421	* @param pVmxTransient Pointer to the VMX transient structure.
422	*
423	* @remarks No-long-jump zone!!!
424	*/
425	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
426	{
427	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
428	AssertRCReturn(rc, rc);
429	return VINF_SUCCESS;
430	}
431
432
433	/**
434	* Reads the VM-entry exception error code field from the VMCS into
435	* the VMX transient structure.
436	*
437	* @returns VBox status code.
438	* @param pVmxTransient Pointer to the VMX transient structure.
439	*
440	* @remarks No-long-jump zone!!!
441	*/
442	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
443	{
444	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
445	AssertRCReturn(rc, rc);
446	return VINF_SUCCESS;
447	}
448
449
450	/**
451	* Reads the VM-exit interruption-information field from the VMCS into the VMX
452	* transient structure.
453	*
454	* @returns VBox status code.
455	* @param pVmxTransient Pointer to the VMX transient structure.
456	*/
457	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
458	{
459	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
460	{
461	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
462	AssertRCReturn(rc, rc);
463	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
464	}
465	return VINF_SUCCESS;
466	}
467
468
469	/**
470	* Reads the VM-exit interruption error code from the VMCS into the VMX
471	* transient structure.
472	*
473	* @returns VBox status code.
474	* @param pVmxTransient Pointer to the VMX transient structure.
475	*/
476	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
477	{
478	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
479	{
480	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
481	AssertRCReturn(rc, rc);
482	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
483	}
484	return VINF_SUCCESS;
485	}
486
487
488	/**
489	* Reads the VM-exit instruction length field from the VMCS into the VMX
490	* transient structure.
491	*
492	* @returns VBox status code.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*/
495	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
496	{
497	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
500	AssertRCReturn(rc, rc);
501	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
502	}
503	return VINF_SUCCESS;
504	}
505
506
507	/**
508	* Reads the exit qualification from the VMCS into the VMX transient structure.
509	*
510	* @returns VBox status code.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
516	{
517	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the IDT-vectoring information field from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVmxTransient Pointer to the VMX transient structure.
531	*
532	* @remarks No-long-jump zone!!!
533	*/
534	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
535	{
536	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
537	{
538	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
539	AssertRCReturn(rc, rc);
540	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
541	}
542	return VINF_SUCCESS;
543	}
544
545
546	/**
547	* Reads the IDT-vectoring error code from the VMCS into the VMX
548	* transient structure.
549	*
550	* @returns VBox status code.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*/
553	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
554	{
555	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
558	AssertRCReturn(rc, rc);
559	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
560	}
561	return VINF_SUCCESS;
562	}
563
564
565	/**
566	* Enters VMX root mode operation on the current CPU.
567	*
568	* @returns VBox status code.
569	* @param pVM Pointer to the VM (optional, can be NULL, after
570	* a resume).
571	* @param HCPhysCpuPage Physical address of the VMXON region.
572	* @param pvCpuPage Pointer to the VMXON region.
573	*/
574	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
575	{
576	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
577	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
578
579	if (pVM)
580	{
581	/* Write the VMCS revision dword to the VMXON region. */
582	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
583	}
584
585	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
586	RTCCUINTREG fFlags = ASMIntDisableFlags();
587
588	/* Enable the VMX bit in CR4. */
589	RTCCUINTREG uCr4 = ASMGetCR4();
590	if (!(uCr4 & X86_CR4_VMXE))
591	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
592
593	/* Enter VMXON root mode. */
594	int rc = VMXEnable(HCPhysCpuPage);
595
596	/* Restore interrupts. */
597	ASMSetFlags(fFlags);
598	return rc;
599	}
600
601
602	/**
603	* Exits VMX root mode operation on the current CPU.
604	*
605	* @returns VBox status code.
606	*/
607	static int hmR0VmxLeaveRootMode(void)
608	{
609	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
610	RTCCUINTREG fFlags = ASMIntDisableFlags();
611	int rc = VINF_SUCCESS;
612
613	/* If we're for some reason not in VMX root mode, then don't leave it. */
614	if (ASMGetCR4() & X86_CR4_VMXE)
615	{
616	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
617	VMXDisable();
618	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
619	}
620	else
621	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
622
623	/* Restore interrupts. */
624	ASMSetFlags(fFlags);
625	return rc;
626	}
627
628
629	/**
630	* Allocates and maps one physically contiguous page. The allocated page is
631	* zero'd out. (Used by various VT-x structures).
632	*
633	* @returns IPRT status code.
634	* @param pMemObj Pointer to the ring-0 memory object.
635	* @param ppVirt Where to store the virtual address of the
636	* allocation.
637	* @param pPhys Where to store the physical address of the
638	* allocation.
639	*/
640	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
641	{
642	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
643	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
644	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
645
646	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
647	if (RT_FAILURE(rc))
648	return rc;
649	ppVirt = RTR0MemObjAddress(pMemObj);
650	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
651	ASMMemZero32(*ppVirt, PAGE_SIZE);
652	return VINF_SUCCESS;
653	}
654
655
656	/**
657	* Frees and unmaps an allocated physical page.
658	*
659	* @param pMemObj Pointer to the ring-0 memory object.
660	* @param ppVirt Where to re-initialize the virtual address of
661	* allocation as 0.
662	* @param pHCPhys Where to re-initialize the physical address of the
663	* allocation as 0.
664	*/
665	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
666	{
667	AssertPtr(pMemObj);
668	AssertPtr(ppVirt);
669	AssertPtr(pHCPhys);
670	if (*pMemObj != NIL_RTR0MEMOBJ)
671	{
672	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
673	AssertRC(rc);
674	*pMemObj = NIL_RTR0MEMOBJ;
675	*ppVirt = 0;
676	*pHCPhys = 0;
677	}
678	}
679
680
681	/**
682	* Worker function to free VT-x related structures.
683	*
684	* @returns IPRT status code.
685	* @param pVM Pointer to the VM.
686	*/
687	static void hmR0VmxStructsFree(PVM pVM)
688	{
689	for (VMCPUID i = 0; i < pVM->cCpus; i++)
690	{
691	PVMCPU pVCpu = &pVM->aCpus[i];
692	AssertPtr(pVCpu);
693
694	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
695	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
696
697	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
698	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
699
700	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
701	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
702	}
703
704	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
705	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
706	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
707	#endif
708	}
709
710
711	/**
712	* Worker function to allocate VT-x related VM structures.
713	*
714	* @returns IPRT status code.
715	* @param pVM Pointer to the VM.
716	*/
717	static int hmR0VmxStructsAlloc(PVM pVM)
718	{
719	/*
720	* Initialize members up-front so we can cleanup properly on allocation failure.
721	*/
722	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
723	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
724	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
725	pVM->hm.s.vmx.HCPhys##a_Name = 0;
726
727	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
728	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
729	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
730	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
731
732	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
733	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
734	#endif
735	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
736
737	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
738	for (VMCPUID i = 0; i < pVM->cCpus; i++)
739	{
740	PVMCPU pVCpu = &pVM->aCpus[i];
741	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
742	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
743	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
744	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
745	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
746	}
747	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
748	#undef VMXLOCAL_INIT_VM_MEMOBJ
749
750	/*
751	* Allocate all the VT-x structures.
752	*/
753	int rc = VINF_SUCCESS;
754	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
755	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
756	if (RT_FAILURE(rc))
757	goto cleanup;
758	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
759	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
760	#endif
761
762	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
763	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
764	{
765	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
766	&pVM->hm.s.vmx.HCPhysApicAccess);
767	if (RT_FAILURE(rc))
768	goto cleanup;
769	}
770
771	/*
772	* Initialize per-VCPU VT-x structures.
773	*/
774	for (VMCPUID i =0; i < pVM->cCpus; i++)
775	{
776	PVMCPU pVCpu = &pVM->aCpus[i];
777	AssertPtr(pVCpu);
778
779	/* Allocate the VM control structure (VMCS). */
780	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
781	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
782	if (RT_FAILURE(rc))
783	goto cleanup;
784
785	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
786	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
787	{
788	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
789	&pVCpu->hm.s.vmx.HCPhysVirtApic);
790	if (RT_FAILURE(rc))
791	goto cleanup;
792	}
793
794	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
795	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
796	{
797	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
798	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
799	if (RT_FAILURE(rc))
800	goto cleanup;
801	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
802	}
803
804	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
805	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
806	if (RT_FAILURE(rc))
807	goto cleanup;
808
809	/* Allocate the VM-exit MSR-load page for the host MSRs. */
810	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
811	if (RT_FAILURE(rc))
812	goto cleanup;
813	}
814
815	return VINF_SUCCESS;
816
817	cleanup:
818	hmR0VmxStructsFree(pVM);
819	return rc;
820	}
821
822
823	/**
824	* Does global VT-x initialization (called during module initialization).
825	*
826	* @returns VBox status code.
827	*/
828	VMMR0DECL(int) VMXR0GlobalInit(void)
829	{
830	/* Setup the main VM exit handlers. */
831	/** @todo I'm told lookup table for function pointers is a bad idea in terms
832	* of performance. Make these static for now and not do change anything
833	* at runtime so we can easily switch to a switch-case approach later. */
834	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
835	#ifdef DEBUG
836	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
837	Assert(s_apfnVMExitHandlers[i]);
838	#endif
839	return VINF_SUCCESS;
840	}
841
842
843	/**
844	* Does global VT-x termination (called during module termination).
845	*/
846	VMMR0DECL(void) VMXR0GlobalTerm()
847	{
848	/* Nothing to do currently. */
849	}
850
851
852	/**
853	* Sets up and activates VT-x on the current CPU.
854	*
855	* @returns VBox status code.
856	* @param pCpu Pointer to the global CPU info struct.
857	* @param pVM Pointer to the VM (can be NULL after a host resume
858	* operation).
859	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
860	* fEnabledByHost is true).
861	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
862	* @a fEnabledByHost is true).
863	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
864	* enable VT-x/AMD-V on the host.
865	*/
866	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
867	{
868	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
869	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
870
871	if (!fEnabledByHost)
872	{
873	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
874	if (RT_FAILURE(rc))
875	return rc;
876	}
877
878	/*
879	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
880	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
881	* each time while reusing a VPID after hitting the MaxASID limit once.
882	*/
883	if ( pVM
884	&& pVM->hm.s.vmx.fVpid
885	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
886	{
887	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
888	pCpu->fFlushAsidBeforeUse = false;
889	}
890	else
891	pCpu->fFlushAsidBeforeUse = true;
892
893	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
894	++pCpu->cTlbFlushes;
895
896	return VINF_SUCCESS;
897	}
898
899
900	/**
901	* Deactivates VT-x on the current CPU.
902	*
903	* @returns VBox status code.
904	* @param pCpu Pointer to the global CPU info struct.
905	* @param pvCpuPage Pointer to the VMXON region.
906	* @param HCPhysCpuPage Physical address of the VMXON region.
907	*/
908	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
909	{
910	NOREF(pCpu);
911	NOREF(pvCpuPage);
912	NOREF(HCPhysCpuPage);
913
914	hmR0VmxLeaveRootMode();
915	return VINF_SUCCESS;
916	}
917
918
919	/**
920	* Sets the permission bits for the specified MSR in the MSR bitmap.
921	*
922	* @param pVCpu Pointer to the VMCPU.
923	* @param ulMSR The MSR value.
924	* @param enmRead Whether reading this MSR causes a VM-exit.
925	* @param enmWrite Whether writing this MSR causes a VM-exit.
926	*/
927	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
928	{
929	unsigned ulBit;
930	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
931
932	/*
933	* Layout:
934	* 0x000 - 0x3ff - Low MSR read bits
935	* 0x400 - 0x7ff - High MSR read bits
936	* 0x800 - 0xbff - Low MSR write bits
937	* 0xc00 - 0xfff - High MSR write bits
938	*/
939	if (ulMsr <= 0x00001FFF)
940	{
941	/* Pentium-compatible MSRs */
942	ulBit = ulMsr;
943	}
944	else if ( ulMsr >= 0xC0000000
945	&& ulMsr <= 0xC0001FFF)
946	{
947	/* AMD Sixth Generation x86 Processor MSRs */
948	ulBit = (ulMsr - 0xC0000000);
949	pbMsrBitmap += 0x400;
950	}
951	else
952	{
953	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
954	return;
955	}
956
957	Assert(ulBit <= 0x1fff);
958	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
959	ASMBitSet(pbMsrBitmap, ulBit);
960	else
961	ASMBitClear(pbMsrBitmap, ulBit);
962
963	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
964	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
965	else
966	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
967	}
968
969
970	/**
971	* Flushes the TLB using EPT.
972	*
973	* @returns VBox status code.
974	* @param pVM Pointer to the VM.
975	* @param pVCpu Pointer to the VMCPU.
976	* @param enmFlush Type of flush.
977	*/
978	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
979	{
980	AssertPtr(pVM);
981	Assert(pVM->hm.s.fNestedPaging);
982
983	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
984
985	uint64_t descriptor[2];
986	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
987	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
988
989	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
990	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
991	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
992	}
993
994
995	/**
996	* Flushes the TLB using VPID.
997	*
998	* @returns VBox status code.
999	* @param pVM Pointer to the VM.
1000	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1001	* enmFlush).
1002	* @param enmFlush Type of flush.
1003	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1004	* on @a enmFlush).
1005	*/
1006	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1007	{
1008	AssertPtr(pVM);
1009	Assert(pVM->hm.s.vmx.fVpid);
1010
1011	uint64_t descriptor[2];
1012	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1013	{
1014	descriptor[0] = 0;
1015	descriptor[1] = 0;
1016	}
1017	else
1018	{
1019	AssertPtr(pVCpu);
1020	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1021	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1022	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1023	descriptor[1] = GCPtr;
1024	}
1025
1026	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1027	AssertMsg(rc == VINF_SUCCESS,
1028	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1029	if ( RT_SUCCESS(rc)
1030	&& pVCpu)
1031	{
1032	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1033	}
1034	}
1035
1036
1037	/**
1038	* Invalidates a guest page by guest virtual address. Only relevant for
1039	* EPT/VPID, otherwise there is nothing really to invalidate.
1040	*
1041	* @returns VBox status code.
1042	* @param pVM Pointer to the VM.
1043	* @param pVCpu Pointer to the VMCPU.
1044	* @param GCVirt Guest virtual address of the page to invalidate.
1045	*/
1046	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1047	{
1048	AssertPtr(pVM);
1049	AssertPtr(pVCpu);
1050	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1051
1052	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1053	if (!fFlushPending)
1054	{
1055	/*
1056	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1057	* See @bugref{6043} and @bugref{6177}.
1058	*
1059	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1060	* function maybe called in a loop with individual addresses.
1061	*/
1062	if (pVM->hm.s.vmx.fVpid)
1063	{
1064	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1065	{
1066	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1067	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1068	}
1069	else
1070	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1071	}
1072	else if (pVM->hm.s.fNestedPaging)
1073	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1074	}
1075
1076	return VINF_SUCCESS;
1077	}
1078
1079
1080	/**
1081	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1082	* otherwise there is nothing really to invalidate.
1083	*
1084	* @returns VBox status code.
1085	* @param pVM Pointer to the VM.
1086	* @param pVCpu Pointer to the VMCPU.
1087	* @param GCPhys Guest physical address of the page to invalidate.
1088	*/
1089	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1090	{
1091	LogFlowFunc(("%RGp\n", GCPhys));
1092
1093	/*
1094	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1095	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1096	* This function might be called in a loop.
1097	*/
1098	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1099	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1100	return VINF_SUCCESS;
1101	}
1102
1103
1104	/**
1105	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1106	* case where neither EPT nor VPID is supported by the CPU.
1107	*
1108	* @param pVM Pointer to the VM.
1109	* @param pVCpu Pointer to the VMCPU.
1110	*
1111	* @remarks Called with interrupts disabled.
1112	*/
1113	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1114	{
1115	NOREF(pVM);
1116	AssertPtr(pVCpu);
1117	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1118	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1119
1120	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1121	AssertPtr(pCpu);
1122
1123	pVCpu->hm.s.TlbShootdown.cPages = 0;
1124	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1125	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1126	pVCpu->hm.s.fForceTLBFlush = false;
1127	return;
1128	}
1129
1130
1131	/**
1132	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1133	*
1134	* @param pVM Pointer to the VM.
1135	* @param pVCpu Pointer to the VMCPU.
1136	* @remarks All references to "ASID" in this function pertains to "VPID" in
1137	* Intel's nomenclature. The reason is, to avoid confusion in compare
1138	* statements since the host-CPU copies are named "ASID".
1139	*
1140	* @remarks Called with interrupts disabled.
1141	* @todo Statistics.
1142	*/
1143	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1144	{
1145	AssertPtr(pVM);
1146	AssertPtr(pVCpu);
1147	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1148	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1149	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1150
1151	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1152	AssertPtr(pCpu);
1153
1154	/*
1155	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1156	* This can happen both for start & resume due to long jumps back to ring-3.
1157	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1158	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1159	*/
1160	bool fNewASID = false;
1161	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1162	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1163	{
1164	pVCpu->hm.s.fForceTLBFlush = true;
1165	fNewASID = true;
1166	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1167	}
1168
1169	/*
1170	* Check for explicit TLB shootdowns.
1171	*/
1172	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1173	{
1174	pVCpu->hm.s.fForceTLBFlush = true;
1175	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1176	}
1177
1178	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1179	if (pVCpu->hm.s.fForceTLBFlush)
1180	{
1181	if (fNewASID)
1182	{
1183	++pCpu->uCurrentAsid;
1184	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1185	{
1186	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1187	pCpu->cTlbFlushes++;
1188	pCpu->fFlushAsidBeforeUse = true;
1189	}
1190
1191	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1192	if (pCpu->fFlushAsidBeforeUse)
1193	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1194	}
1195	else
1196	{
1197	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1198	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1199	else
1200	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1201	}
1202
1203	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1204	pVCpu->hm.s.fForceTLBFlush = false;
1205	}
1206	else
1207	{
1208	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1209	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1210	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1211	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1212
1213	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1214	* not be executed. See hmQueueInvlPage() where it is commented
1215	* out. Support individual entry flushing someday. */
1216	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1217	{
1218	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1219
1220	/*
1221	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1222	* as supported by the CPU.
1223	*/
1224	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1225	{
1226	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1227	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1228	}
1229	else
1230	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1231	}
1232	else
1233	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1234	}
1235	pVCpu->hm.s.TlbShootdown.cPages = 0;
1236	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1237
1238	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1239	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1240	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1241	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1242	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1243	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1244
1245	/* Update VMCS with the VPID. */
1246	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1247	AssertRC(rc);
1248	}
1249
1250
1251	/**
1252	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1253	*
1254	* @returns VBox status code.
1255	* @param pVM Pointer to the VM.
1256	* @param pVCpu Pointer to the VMCPU.
1257	*
1258	* @remarks Called with interrupts disabled.
1259	* @todo Statistics.
1260	*/
1261	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1262	{
1263	AssertPtr(pVM);
1264	AssertPtr(pVCpu);
1265	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1266	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1267
1268	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1269	AssertPtr(pCpu);
1270
1271	/*
1272	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1273	* This can happen both for start & resume due to long jumps back to ring-3.
1274	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1275	*/
1276	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1277	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1278	{
1279	pVCpu->hm.s.fForceTLBFlush = true;
1280	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1281	}
1282
1283	/* Check for explicit TLB shootdown flushes. */
1284	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1285	{
1286	pVCpu->hm.s.fForceTLBFlush = true;
1287	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1288	}
1289
1290	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1291	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1292
1293	if (pVCpu->hm.s.fForceTLBFlush)
1294	{
1295	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1296	pVCpu->hm.s.fForceTLBFlush = false;
1297	}
1298	else
1299	{
1300	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1301	* not be executed. See hmQueueInvlPage() where it is commented
1302	* out. Support individual entry flushing someday. */
1303	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1304	{
1305	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1306	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1307	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1308	}
1309	else
1310	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1311	}
1312
1313	pVCpu->hm.s.TlbShootdown.cPages= 0;
1314	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1315	}
1316
1317
1318	/**
1319	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1320	*
1321	* @returns VBox status code.
1322	* @param pVM Pointer to the VM.
1323	* @param pVCpu Pointer to the VMCPU.
1324	*
1325	* @remarks Called with interrupts disabled.
1326	* @todo Statistics.
1327	*/
1328	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1329	{
1330	AssertPtr(pVM);
1331	AssertPtr(pVCpu);
1332	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1333	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1334
1335	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1336
1337	/*
1338	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1339	* This can happen both for start & resume due to long jumps back to ring-3.
1340	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1341	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1342	*/
1343	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1344	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1345	{
1346	pVCpu->hm.s.fForceTLBFlush = true;
1347	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1348	}
1349
1350	/* Check for explicit TLB shootdown flushes. */
1351	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1352	{
1353	/*
1354	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1355	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1356	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1357	*/
1358	pVCpu->hm.s.fForceTLBFlush = true;
1359	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1360	}
1361
1362	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1363	if (pVCpu->hm.s.fForceTLBFlush)
1364	{
1365	++pCpu->uCurrentAsid;
1366	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1367	{
1368	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1369	pCpu->fFlushAsidBeforeUse = true;
1370	pCpu->cTlbFlushes++;
1371	}
1372
1373	pVCpu->hm.s.fForceTLBFlush = false;
1374	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1375	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1376	if (pCpu->fFlushAsidBeforeUse)
1377	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1378	}
1379	else
1380	{
1381	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1382	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1383	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1384	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1385
1386	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1387	* not be executed. See hmQueueInvlPage() where it is commented
1388	* out. Support individual entry flushing someday. */
1389	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1390	{
1391	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1392	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1393	{
1394	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1395	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1396	}
1397	else
1398	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1399	}
1400	else
1401	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1402	}
1403
1404	pVCpu->hm.s.TlbShootdown.cPages = 0;
1405	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1406
1407	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1408	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1409	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1410	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1411	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1412	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1413
1414	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1415	AssertRC(rc);
1416	}
1417
1418
1419	/**
1420	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1421	* TLB entries from the host TLB before VM-entry.
1422	*
1423	* @returns VBox status code.
1424	* @param pVM Pointer to the VM.
1425	*/
1426	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1427	{
1428	/*
1429	* Determine optimal flush type for nested paging.
1430	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1431	* guest execution (see hmR3InitFinalizeR0()).
1432	*/
1433	if (pVM->hm.s.fNestedPaging)
1434	{
1435	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1436	{
1437	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1438	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1439	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1440	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1441	else
1442	{
1443	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1444	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1445	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1446	}
1447
1448	/* Make sure the write-back cacheable memory type for EPT is supported. */
1449	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1450	{
1451	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1452	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1453	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1454	}
1455	}
1456	else
1457	{
1458	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1459	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1460	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1461	}
1462	}
1463
1464	/*
1465	* Determine optimal flush type for VPID.
1466	*/
1467	if (pVM->hm.s.vmx.fVpid)
1468	{
1469	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1470	{
1471	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1472	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1473	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1474	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1475	else
1476	{
1477	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1478	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1479	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1480	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1481	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1482	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1483	pVM->hm.s.vmx.fVpid = false;
1484	}
1485	}
1486	else
1487	{
1488	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1489	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1490	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1491	pVM->hm.s.vmx.fVpid = false;
1492	}
1493	}
1494
1495	/*
1496	* Setup the handler for flushing tagged-TLBs.
1497	*/
1498	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1499	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1500	else if (pVM->hm.s.fNestedPaging)
1501	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1502	else if (pVM->hm.s.vmx.fVpid)
1503	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1504	else
1505	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1506	return VINF_SUCCESS;
1507	}
1508
1509
1510	/**
1511	* Sets up pin-based VM-execution controls in the VMCS.
1512	*
1513	* @returns VBox status code.
1514	* @param pVM Pointer to the VM.
1515	* @param pVCpu Pointer to the VMCPU.
1516	*/
1517	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1518	{
1519	AssertPtr(pVM);
1520	AssertPtr(pVCpu);
1521
1522	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1523	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1524
1525	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1526	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1527	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1528
1529	/* Enable the VMX preemption timer. */
1530	if (pVM->hm.s.vmx.fUsePreemptTimer)
1531	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1532
1533	if ((val & zap) != val)
1534	{
1535	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1536	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1537	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1538	}
1539
1540	val &= zap;
1541	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1542	AssertRCReturn(rc, rc);
1543
1544	/* Update VCPU with the currently set pin-based VM-execution controls. */
1545	pVCpu->hm.s.vmx.u32PinCtls = val;
1546	return rc;
1547	}
1548
1549
1550	/**
1551	* Sets up processor-based VM-execution controls in the VMCS.
1552	*
1553	* @returns VBox status code.
1554	* @param pVM Pointer to the VM.
1555	* @param pVMCPU Pointer to the VMCPU.
1556	*/
1557	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1558	{
1559	AssertPtr(pVM);
1560	AssertPtr(pVCpu);
1561
1562	int rc = VERR_INTERNAL_ERROR_5;
1563	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1564	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1565
1566	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1567	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1568	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1569	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1570	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1571	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1572	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1573
1574	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1575	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1576	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1577	{
1578	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1579	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1580	}
1581
1582	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1583	if (!pVM->hm.s.fNestedPaging)
1584	{
1585	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1586	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1587	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1588	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1589	}
1590
1591	/* Use TPR shadowing if supported by the CPU. */
1592	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1593	{
1594	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1595	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1596	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1597	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1598
1599	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1600	/* CR8 writes causes a VM-exit based on TPR threshold. */
1601	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1602	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1603	}
1604	else
1605	{
1606	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1607	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1608	}
1609
1610	/* Use MSR-bitmaps if supported by the CPU. */
1611	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1612	{
1613	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1614
1615	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1616	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1617	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1618	AssertRCReturn(rc, rc);
1619
1620	/*
1621	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1622	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1623	*/
1624	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1625	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1626	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1627	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1628	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1629	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1630	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1631	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1632	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1633	}
1634
1635	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1636	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1637	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1638
1639	if ((val & zap) != val)
1640	{
1641	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1642	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1643	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1644	}
1645
1646	val &= zap;
1647	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1648	AssertRCReturn(rc, rc);
1649
1650	/* Update VCPU with the currently set processor-based VM-execution controls. */
1651	pVCpu->hm.s.vmx.u32ProcCtls = val;
1652
1653	/*
1654	* Secondary processor-based VM-execution controls.
1655	*/
1656	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1657	{
1658	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1659	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1660
1661	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1662
1663	if (pVM->hm.s.fNestedPaging)
1664	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1665	else
1666	{
1667	/*
1668	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1669	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1670	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1671	*/
1672	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1673	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1674	}
1675
1676	if (pVM->hm.s.vmx.fVpid)
1677	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1678
1679	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1680	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1681
1682	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1683	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1684	* done dynamically. */
1685	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1686	{
1687	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1688	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1689	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1690	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1691	AssertRCReturn(rc, rc);
1692	}
1693
1694	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1695	{
1696	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1697	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1698	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1699	}
1700
1701	if ((val & zap) != val)
1702	{
1703	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1704	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1705	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1706	}
1707
1708	val &= zap;
1709	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1710	AssertRCReturn(rc, rc);
1711
1712	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1713	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1714	}
1715
1716	return VINF_SUCCESS;
1717	}
1718
1719
1720	/**
1721	* Sets up miscellaneous (everything other than Pin & Processor-based
1722	* VM-execution) control fields in the VMCS.
1723	*
1724	* @returns VBox status code.
1725	* @param pVM Pointer to the VM.
1726	* @param pVCpu Pointer to the VMCPU.
1727	*/
1728	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1729	{
1730	AssertPtr(pVM);
1731	AssertPtr(pVCpu);
1732
1733	int rc = VERR_GENERAL_FAILURE;
1734
1735	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1736	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1737
1738	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1739
1740	/*
1741	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1742	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1743	* We thus use the exception bitmap to control it rather than use both.
1744	*/
1745	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1746	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1747
1748	/** @todo Explore possibility of using IO-bitmaps. */
1749	/* All IO & IOIO instructions cause VM-exits. */
1750	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1751	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1752
1753	/* Setup MSR autoloading/autostoring. */
1754	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1755	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1756	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1757	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1758	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1759	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1760
1761	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1762	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1763	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1764	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1765
1766	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1767	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1768
1769	/* Setup debug controls */
1770	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1771	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1772	AssertRCReturn(rc, rc);
1773	return rc;
1774	}
1775
1776
1777	/**
1778	* Sets up the initial exception bitmap in the VMCS based on static conditions
1779	* (i.e. conditions that cannot ever change at runtime).
1780	*
1781	* @returns VBox status code.
1782	* @param pVM Pointer to the VM.
1783	* @param pVCpu Pointer to the VMCPU.
1784	*/
1785	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1786	{
1787	AssertPtr(pVM);
1788	AssertPtr(pVCpu);
1789
1790	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1791
1792	uint32_t u32XcptBitmap = 0;
1793
1794	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1795	if (!pVM->hm.s.fNestedPaging)
1796	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1797
1798	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1799	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1800	AssertRCReturn(rc, rc);
1801	return rc;
1802	}
1803
1804
1805	/**
1806	* Sets up the initial guest-state mask. The guest-state mask is consulted
1807	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1808	* for the nested virtualization case (as it would cause a VM-exit).
1809	*
1810	* @param pVCpu Pointer to the VMCPU.
1811	*/
1812	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1813	{
1814	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1815	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1816	return VINF_SUCCESS;
1817	}
1818
1819
1820	/**
1821	* Does per-VM VT-x initialization.
1822	*
1823	* @returns VBox status code.
1824	* @param pVM Pointer to the VM.
1825	*/
1826	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1827	{
1828	LogFlowFunc(("pVM=%p\n", pVM));
1829
1830	int rc = hmR0VmxStructsAlloc(pVM);
1831	if (RT_FAILURE(rc))
1832	{
1833	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1834	return rc;
1835	}
1836
1837	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1838	{
1839	PVMCPU pVCpu = &pVM->aCpus[i];
1840
1841	/* Current guest paging mode. */
1842	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1843	}
1844
1845	return VINF_SUCCESS;
1846	}
1847
1848
1849	/**
1850	* Does per-VM VT-x termination.
1851	*
1852	* @returns VBox status code.
1853	* @param pVM Pointer to the VM.
1854	*/
1855	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1856	{
1857	LogFlowFunc(("pVM=%p\n", pVM));
1858
1859	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1860	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1861	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1862	#endif
1863	hmR0VmxStructsFree(pVM);
1864	return VINF_SUCCESS;
1865	}
1866
1867
1868	/**
1869	* Sets up the VM for execution under VT-x.
1870	* This function is only called once per-VM during initalization.
1871	*
1872	* @returns VBox status code.
1873	* @param pVM Pointer to the VM.
1874	*/
1875	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1876	{
1877	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1878	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1879
1880	LogFlowFunc(("pVM=%p\n", pVM));
1881
1882	/*
1883	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1884	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1885	*/
1886	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1887	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1888	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1889	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1890	{
1891	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1892	return VERR_INTERNAL_ERROR;
1893	}
1894
1895	/* Initialize these always, see hmR3InitFinalizeR0().*/
1896	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1897	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1898
1899	/* Setup the tagged-TLB flush handlers. */
1900	int rc = hmR0VmxSetupTaggedTlb(pVM);
1901	if (RT_FAILURE(rc))
1902	{
1903	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1904	return rc;
1905	}
1906
1907	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1908	{
1909	PVMCPU pVCpu = &pVM->aCpus[i];
1910	AssertPtr(pVCpu);
1911	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1912
1913	/* Set revision dword at the beginning of the VMCS structure. */
1914	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1915
1916	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1917	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1918	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1919	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1920
1921	/* Load this VMCS as the current VMCS. */
1922	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1923	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1924	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1925
1926	/* Setup the pin-based VM-execution controls. */
1927	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1928	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1929	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1930
1931	/* Setup the processor-based VM-execution controls. */
1932	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1933	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1934	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1935
1936	/* Setup the rest (miscellaneous) VM-execution controls. */
1937	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Setup the initial exception bitmap. */
1942	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the initial guest-state mask. */
1947	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1952	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1953	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1954	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1955	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1956	#endif
1957
1958	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1959	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1960	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1961	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1962
1963	/* Update the last error record for this VCPU. */
1964	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1965	}
1966
1967	return VINF_SUCCESS;
1968	}
1969
1970
1971	/**
1972	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1973	* the VMCS.
1974	*
1975	* @returns VBox status code.
1976	* @param pVM Pointer to the VM.
1977	* @param pVCpu Pointer to the VMCPU.
1978	*/
1979	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
1980	{
1981	RTCCUINTREG uReg = ASMGetCR0();
1982	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
1983	AssertRCReturn(rc, rc);
1984
1985	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1986	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
1987	if (VMX_IS_64BIT_HOST_MODE())
1988	{
1989	uint64_t uReg = hmR0Get64bitCR3();
1990	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
1991	}
1992	else
1993	#endif
1994	{
1995	uReg = ASMGetCR3();
1996	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
1997	}
1998	AssertRCReturn(rc, rc);
1999
2000	uReg = ASMGetCR4();
2001	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2002	AssertRCReturn(rc, rc);
2003	return rc;
2004	}
2005
2006
2007	/**
2008	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2009	* the host-state area in the VMCS.
2010	*
2011	* @returns VBox status code.
2012	* @param pVM Pointer to the VM.
2013	* @param pVCpu Pointer to the VMCPU.
2014	*/
2015	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2016	{
2017	int rc = VERR_INTERNAL_ERROR_5;
2018	RTSEL uSelCS = 0;
2019	RTSEL uSelSS = 0;
2020	RTSEL uSelDS = 0;
2021	RTSEL uSelES = 0;
2022	RTSEL uSelFS = 0;
2023	RTSEL uSelGS = 0;
2024	RTSEL uSelTR = 0;
2025
2026	/*
2027	* Host Selector registers.
2028	*/
2029	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2030	if (VMX_IS_64BIT_HOST_MODE())
2031	{
2032	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2033	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2034	}
2035	else
2036	{
2037	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2038	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2039	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2040	}
2041	#else
2042	uSelCS = ASMGetCS();
2043	uSelSS = ASMGetSS();
2044	#endif
2045
2046	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2047	/** @todo Verify if we have any platform that actually run with DS or ES with
2048	* RPL != 0 in kernel space. */
2049	uSelDS = 0;
2050	uSelES = 0;
2051	uSelFS = 0;
2052	uSelGS = 0;
2053	uSelTR = ASMGetTR();
2054
2055	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2056	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2057	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2058	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2059	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2060	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2061	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2062	Assert(uSelCS != 0);
2063	Assert(uSelTR != 0);
2064
2065	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2066	#if 0
2067	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2068	Assert(uSelSS != 0);
2069	#endif
2070
2071	/* Write these host selector fields into the host-state area in the VMCS. */
2072	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2073	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2074	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2075	#if 0
2076	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2077	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2078	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2079	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2080	#endif
2081	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2082	AssertRCReturn(rc, rc);
2083
2084	/*
2085	* Host GDTR and IDTR.
2086	*/
2087	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2088	* be safe to -not- save and restore GDTR and IDTR in the assembly
2089	* code and just do it here and don't care if the limits are zapped on
2090	* VM-exit. */
2091	RTGDTR Gdtr;
2092	RT_ZERO(Gdtr);
2093	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2094	if (VMX_IS_64BIT_HOST_MODE())
2095	{
2096	X86XDTR64 Gtr64;
2097	X86XDTR64 Idtr64;
2098	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2099	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2100	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2101	Gdtr.cbGdt = Gdtr64.cb;
2102	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2103	}
2104	else
2105	#endif
2106	{
2107	RTIDTR Idtr;
2108	ASMGetGDTR(&Gdtr);
2109	ASMGetIDTR(&Idtr);
2110	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2111	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2112	}
2113	AssertRCReturn(rc, rc);
2114
2115	/*
2116	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2117	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2118	*/
2119	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2120	{
2121	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2122	return VERR_VMX_INVALID_HOST_STATE;
2123	}
2124
2125	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2126	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2127	if (VMX_IS_64BIT_HOST_MODE())
2128	{
2129	/* We need the 64-bit TR base for hybrid darwin. */
2130	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2131	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2132	}
2133	else
2134	#endif
2135	{
2136	uintptr_t uTRBase;
2137	#if HC_ARCH_BITS == 64
2138	uTRBase = X86DESC64_BASE(pDesc);
2139	#else
2140	uTRBase = X86DESC_BASE(pDesc);
2141	#endif
2142	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2143	}
2144	AssertRCReturn(rc, rc);
2145
2146	/*
2147	* Host FS base and GS base.
2148	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2149	* would take care of the bases. In 64-bit, the MSRs come into play.
2150	*/
2151	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2152	if (VMX_IS_64BIT_HOST_MODE())
2153	{
2154	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2155	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2156	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2157	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2158	AssertRCReturn(rc, rc);
2159	}
2160	#endif
2161	return rc;
2162	}
2163
2164
2165	/**
2166	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2167	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2168	* the host after every successful VM exit.
2169	*
2170	* @returns VBox status code.
2171	* @param pVM Pointer to the VM.
2172	* @param pVCpu Pointer to the VMCPU.
2173	*/
2174	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2175	{
2176	AssertPtr(pVCpu);
2177	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2178
2179	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2180	unsigned idxHostMsr = 0;
2181	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2182
2183	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2184	{
2185	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2186	pHostMsr->u32Reserved = 0;
2187	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2188	if (CPUMIsGuestInLongMode(pVCpu))
2189	{
2190	/* Must match the EFER value in our 64 bits switcher. */
2191	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2192	}
2193	else
2194	#endif
2195	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2196	pHostMsr++; idxHostMsr++;
2197	}
2198
2199	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2200	if (VMX_IS_64BIT_HOST_MODE())
2201	{
2202	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2203	pHostMsr->u32Reserved = 0;
2204	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2205	pHostMsr++; idxHostMsr++;
2206	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2207	pHostMsr->u32Reserved = 0;
2208	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2209	pHostMsr++; idxHostMsr++;
2210	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2211	pHostMsr->u32Reserved = 0;
2212	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2213	pHostMsr++; idxHostMsr++;
2214	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2215	#if 0
2216	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2217	pMsr->u32Reserved = 0;
2218	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2219	pHostMsr++; idxHostMsr++;
2220	#endif
2221	}
2222	#endif
2223
2224	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2225	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2226	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227
2228	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2229	AssertRCReturn(rc, rc);
2230
2231	/*
2232	* Host Sysenter MSRs.
2233	*/
2234	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2235	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2236	if (VMX_IS_64BIT_HOST_MODE())
2237	{
2238	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2239	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2240	}
2241	else
2242	{
2243	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2244	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2245	}
2246	#elif HC_ARCH_BITS == 32
2247	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2248	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2249	#else
2250	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2251	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2252	#endif
2253	AssertRCReturn(rc, rc);
2254
2255	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2256	* hmR0VmxSetupExitCtls() !! */
2257	return rc;
2258	}
2259
2260
2261	/**
2262	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2263	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2264	* controls".
2265	*
2266	* @returns VBox status code.
2267	* @param pVM Pointer to the VM.
2268	* @param pVCpu Pointer to the VMCPU.
2269	* @param pCtx Pointer to the guest-CPU context.
2270	*
2271	* @remarks No-long-jump zone!!!
2272	*/
2273	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2274	{
2275	int rc = VINF_SUCCESS;
2276	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2277	{
2278	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2279	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2280
2281	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2282	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2283
2284	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2285	if (CPUMIsGuestInLongModeEx(pCtx))
2286	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2287	else
2288	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2289
2290	/*
2291	* The following should not be set (since we're not in SMM mode):
2292	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2293	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2294	*/
2295
2296	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2297	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2298	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2299
2300	if ((val & zap) != val)
2301	{
2302	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2303	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2304	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2305	}
2306
2307	val &= zap;
2308	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2309	AssertRCReturn(rc, rc);
2310
2311	/* Update VCPU with the currently set VM-exit controls. */
2312	pVCpu->hm.s.vmx.u32EntryCtls = val;
2313	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2314	}
2315	return rc;
2316	}
2317
2318
2319	/**
2320	* Sets up the VM-exit controls in the VMCS.
2321	*
2322	* @returns VBox status code.
2323	* @param pVM Pointer to the VM.
2324	* @param pVCpu Pointer to the VMCPU.
2325	*/
2326	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2327	{
2328	int rc = VINF_SUCCESS;
2329	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2330	{
2331	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2332	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2333
2334	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2335	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2336
2337	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2338	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2339	if (VMX_IS_64BIT_HOST_MODE())
2340	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2341	else
2342	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2343	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2344	if (CPUMIsGuestInLongModeEx(pCtx))
2345	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2346	else
2347	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2348	#endif
2349
2350	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2351	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2352
2353	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2354	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2355	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2356	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2357	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR,
2358	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER. */
2359
2360	if ((val & zap) != val)
2361	{
2362	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2363	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2364	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2365	}
2366
2367	val &= zap;
2368	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2369
2370	/* Update VCPU with the currently set VM-exit controls. */
2371	pVCpu->hm.s.vmx.u32ExitCtls = val;
2372	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2373	}
2374	return rc;
2375	}
2376
2377
2378	/**
2379	* Loads the guest APIC and related state.
2380	*
2381	* @returns VBox status code.
2382	* @param pVM Pointer to the VM.
2383	* @param pVCpu Pointer to the VMCPU.
2384	* @param pCtx Pointer to the guest-CPU context.
2385	*/
2386	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2387	{
2388	int rc = VINF_SUCCESS;
2389	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2390	{
2391	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2392	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2393	{
2394	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2395
2396	bool fPendingIntr = false;
2397	uint8_t u8GuestTpr = 0;
2398	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2399	AssertRCReturn(rc, rc);
2400
2401	/*
2402	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2403	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2404	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2405	* the interrupt when we VM-exit for other reasons.
2406	*/
2407	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2408	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2409	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2410	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2411
2412	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2413	AssertRCReturn(rc, rc);
2414
2415	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2416	if (pVM->hm.s.fTPRPatchingActive)
2417	{
2418	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2419	pCtx->msrLSTAR = u8GuestTpr;
2420	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2421	{
2422	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2423	if (fPendingIntr)
2424	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2425	else
2426	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427	}
2428	}
2429	}
2430
2431	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2432	}
2433	return rc;
2434	}
2435
2436
2437	/**
2438	* Loads the guest's RIP into the guest-state area in the VMCS.
2439	*
2440	* @returns VBox status code.
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2444	* out-of-sync. Make sure to update the required fields
2445	* before using them.
2446	*
2447	* @remarks No-long-jump zone!!!
2448	*/
2449	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2450	{
2451	int rc = VINF_SUCCESS;
2452	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2453	{
2454	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2455	AssertRCReturn(rc, rc);
2456	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2457	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2458	}
2459	return rc;
2460	}
2461
2462
2463	/**
2464	* Loads the guest's RSP into the guest-state area in the VMCS.
2465	*
2466	* @returns VBox status code.
2467	* @param pVM Pointer to the VM.
2468	* @param pVCpu Pointer to the VMCPU.
2469	* @param pCtx Pointer to the guest-CPU context.
2470	*
2471	* @remarks No-long-jump zone!!!
2472	*/
2473	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2474	{
2475	int rc = VINF_SUCCESS;
2476	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2477	{
2478	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2479	AssertRCReturn(rc, rc);
2480	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2481	}
2482	return rc;
2483	}
2484
2485
2486	/**
2487	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2488	*
2489	* @returns VBox status code.
2490	* @param pVM Pointer to the VM.
2491	* @param pVCpu Pointer to the VMCPU.
2492	* @param pCtx Pointer to the guest-CPU context.
2493	*
2494	* @remarks No-long-jump zone!!!
2495	*/
2496	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2497	{
2498	int rc = VINF_SUCCESS;
2499	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2500	{
2501	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2502	Let us assert it as such and use native-width VMWRITE. */
2503	X86RFLAGS uRFlags = pCtx->rflags;
2504	Assert(uRFlags.u64 >> 32 == 0);
2505	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2506	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2507
2508	/*
2509	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2510	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2511	*/
2512	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2513	{
2514	Assert(pVM->hm.s.vmx.pRealModeTSS);
2515	Assert(PDMVmmDevHeapIsEnabled(pVM));
2516	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2517	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2518	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2519	}
2520
2521	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2522	AssertRCReturn(rc, rc);
2523
2524	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2525	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2526	}
2527	return rc;
2528	}
2529
2530
2531	/**
2532	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2533	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2534	* assembly code.
2535	*
2536	* @returns VBox status code.
2537	* @param pVM Pointer to the VM.
2538	* @param pVCpu Pointer to the VMCPU.
2539	* @param pCtx Pointer to the guest-CPU context.
2540	*
2541	* @remarks No-long-jump zone!!!
2542	*/
2543	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2544	{
2545	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2546	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2547	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2548	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2549	AssertRCReturn(rc, rc);
2550	return rc;
2551	}
2552
2553
2554	/**
2555	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2556	* in the VMCS.
2557	*
2558	* @returns VBox status code.
2559	* @param pVM Pointer to the VM.
2560	* @param pVCpu Pointer to the VMCPU.
2561	* @param pCtx Pointer to the guest-CPU context.
2562	*
2563	* @remarks No-long-jump zone!!!
2564	*/
2565	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2566	{
2567	int rc = VINF_SUCCESS;
2568
2569	/*
2570	* Guest CR0.
2571	* Guest FPU.
2572	*/
2573	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2574	{
2575	uint64_t u64GuestCR0 = pCtx->cr0;
2576
2577	/* The guest's view (read access) of its CR0 is unblemished. */
2578	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2579	AssertRCReturn(rc, rc);
2580	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2581
2582	/* Setup VT-x's view of the guest CR0. */
2583	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2584	if (pVM->hm.s.fNestedPaging)
2585	{
2586	if (CPUMIsGuestPagingEnabledEx(pCtx))
2587	{
2588	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2589	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2590	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2591	}
2592	else
2593	{
2594	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2595	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2596	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2597	}
2598
2599	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2600	AssertRCReturn(rc, rc);
2601	}
2602	else
2603	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2604
2605	/*
2606	* Guest FPU bits.
2607	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2608	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2609	*/
2610	u64GuestCR0 \|= X86_CR0_NE;
2611	bool fInterceptNM = false;
2612	if (CPUMIsGuestFPUStateActive(pVCpu))
2613	{
2614	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2615	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2616	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2617	}
2618	else
2619	{
2620	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2621	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2622	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2623	}
2624
2625	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2626	bool fInterceptMF = false;
2627	if (!(pCtx->cr0 & X86_CR0_NE))
2628	fInterceptMF = true;
2629
2630	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2631	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2632	{
2633	Assert(PDMVmmDevHeapIsEnabled(pVM));
2634	Assert(pVM->hm.s.vmx.pRealModeTSS);
2635	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2636	fInterceptNM = true;
2637	fInterceptMF = true;
2638	}
2639	else
2640	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2641
2642	if (fInterceptNM)
2643	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2644	else
2645	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2646
2647	if (fInterceptMF)
2648	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2649	else
2650	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2651
2652	/* Additional intercepts for debugging, define these yourself explicitly. */
2653	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2654	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2655	\| RT_BIT(X86_XCPT_DB)
2656	\| RT_BIT(X86_XCPT_DE)
2657	\| RT_BIT(X86_XCPT_NM)
2658	\| RT_BIT(X86_XCPT_UD)
2659	\| RT_BIT(X86_XCPT_NP)
2660	\| RT_BIT(X86_XCPT_SS)
2661	\| RT_BIT(X86_XCPT_GP)
2662	\| RT_BIT(X86_XCPT_PF)
2663	\| RT_BIT(X86_XCPT_MF);
2664	#elif defined(VBOX_ALWAYS_TRAP_PF)
2665	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2666	#endif
2667
2668	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2669
2670	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2671	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2672	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2673	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2674	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2675	else
2676	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2677
2678	u64GuestCR0 \|= uSetCR0;
2679	u64GuestCR0 &= uZapCR0;
2680	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2681
2682	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2683	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2684	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2685	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2686
2687	/*
2688	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2689	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2690	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2691	*/
2692	uint64_t u64CR0Mask = 0;
2693	u64CR0Mask = X86_CR0_PE
2694	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2695	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2696	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2697	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2698	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2699
2700	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2701	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2702	u64CR0Mask &= ~X86_CR0_PE;
2703
2704	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2705	if (fInterceptNM)
2706	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2707	else
2708	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2709
2710	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2711	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2712	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2713	AssertRCReturn(rc, rc);
2714
2715	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2716	}
2717
2718	/*
2719	* Guest CR2.
2720	* It's always loaded in the assembler code. Nothing to do here.
2721	*/
2722
2723	/*
2724	* Guest CR3.
2725	*/
2726	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2727	{
2728	uint64_t u64GuestCR3 = 0;
2729	if (pVM->hm.s.fNestedPaging)
2730	{
2731	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2732
2733	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2734	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2735	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2736	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2737
2738	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2739	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2740	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2741
2742	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2743	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2744	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2745	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2746
2747	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2748	AssertRCReturn(rc, rc);
2749	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2750
2751	if ( CPUMIsGuestPagingEnabledEx(pCtx)
2752	\|\| pVM->hm.s.vmx.fUnrestrictedGuest)
2753	{
2754	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2755	if (CPUMIsGuestInPAEModeEx(pCtx))
2756	{
2757	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2758	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2759	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2760	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2761	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2762	AssertRCReturn(rc, rc);
2763	}
2764
2765	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2766	have Unrestricted Execution to handle the guest when it's not using paging. */
2767	u64GuestCR3 = pCtx->cr3;
2768	}
2769	else
2770	{
2771	/*
2772	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2773	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2774	* EPT takes care of translating it to host-physical addresses.
2775	*/
2776	RTGCPHYS GCPhys;
2777	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2778	Assert(PDMVmmDevHeapIsEnabled(pVM));
2779
2780	/* We obtain it here every time as the guest could have relocated this PCI region. */
2781	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2782	AssertRCReturn(rc, rc);
2783
2784	u64GuestCR3 = GCPhys;
2785	}
2786	}
2787	else
2788	{
2789	/* Non-nested paging case, just use the hypervisor's CR3. */
2790	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2791	}
2792
2793	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2794	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2795	AssertRCReturn(rc, rc);
2796
2797	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2798	}
2799
2800	/*
2801	* Guest CR4.
2802	*/
2803	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2804	{
2805	uint64_t u64GuestCR4 = pCtx->cr4;
2806
2807	/* The guest's view of its CR4 is unblemished. */
2808	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2809	AssertRCReturn(rc, rc);
2810	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2811
2812	/* Setup VT-x's view of the guest CR4. */
2813	/*
2814	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2815	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2816	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2817	*/
2818	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2819	{
2820	Assert(pVM->hm.s.vmx.pRealModeTSS);
2821	Assert(PDMVmmDevHeapIsEnabled(pVM));
2822	u64GuestCR4 &= ~X86_CR4_VME;
2823	}
2824
2825	if (pVM->hm.s.fNestedPaging)
2826	{
2827	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2828	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2829	{
2830	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2831	u64GuestCR4 \|= X86_CR4_PSE;
2832	/* Our identity mapping is a 32 bits page directory. */
2833	u64GuestCR4 &= ~X86_CR4_PAE;
2834	}
2835	/* else use guest CR4.*/
2836	}
2837	else
2838	{
2839	/*
2840	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2841	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2842	*/
2843	switch (pVCpu->hm.s.enmShadowMode)
2844	{
2845	case PGMMODE_REAL: /* Real-mode. */
2846	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2847	case PGMMODE_32_BIT: /* 32-bit paging. */
2848	{
2849	u64GuestCR4 &= ~X86_CR4_PAE;
2850	break;
2851	}
2852
2853	case PGMMODE_PAE: /* PAE paging. */
2854	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2855	{
2856	u64GuestCR4 \|= X86_CR4_PAE;
2857	break;
2858	}
2859
2860	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2861	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2862	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2863	break;
2864	#endif
2865	default:
2866	AssertFailed();
2867	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2868	}
2869	}
2870
2871	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2872	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2873	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2874	u64GuestCR4 \|= uSetCR4;
2875	u64GuestCR4 &= uZapCR4;
2876
2877	/* Write VT-x's view of the guest CR4 into the VMCS. */
2878	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2879	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2880
2881	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2882	uint64_t u64CR4Mask = 0;
2883	u64CR4Mask = X86_CR4_VME
2884	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2885	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2886	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2887	\| X86_CR4_VMXE;
2888	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2889	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2890	AssertRCReturn(rc, rc);
2891
2892	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2893	}
2894	return rc;
2895	}
2896
2897
2898	/**
2899	* Loads the guest debug registers into the guest-state area in the VMCS.
2900	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2901	*
2902	* @returns VBox status code.
2903	* @param pVM Pointer to the VM.
2904	* @param pVCpu Pointer to the VMCPU.
2905	* @param pCtx Pointer to the guest-CPU context.
2906	*
2907	* @remarks No-long-jump zone!!!
2908	*/
2909	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2910	{
2911	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2912	return VINF_SUCCESS;
2913
2914	#ifdef DEBUG
2915	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2916	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2917	{
2918	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2919	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2920	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2921	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2922	}
2923	#endif
2924
2925	int rc = VERR_INTERNAL_ERROR_5;
2926	bool fInterceptDB = false;
2927	bool fInterceptMovDRx = false;
2928	if (DBGFIsStepping(pVCpu))
2929	{
2930	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2931	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2932	{
2933	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2934	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2935	AssertRCReturn(rc, rc);
2936	Assert(fInterceptDB == false);
2937	}
2938	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2939	{
2940	Assert(fInterceptDB == false);
2941	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2942	* just continuing? */
2943	}
2944	else
2945	fInterceptDB = true;
2946	}
2947	else
2948	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2949
2950	/*
2951	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2952	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2953	* The same for the hypervisor DRx registers, priority is for the guest here.
2954	*/
2955	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
2956	&& !CPUMIsGuestDebugStateActive(pVCpu))
2957	{
2958	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
2959	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2960	AssertRCReturn(rc, rc);
2961	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2962	Assert(fInterceptMovDRx == false);
2963	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2964	}
2965	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
2966	&& !CPUMIsHyperDebugStateActive(pVCpu))
2967	{
2968	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
2969	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2970	AssertRCReturn(rc, rc);
2971	Assert(CPUMIsHyperDebugStateActive(pVCpu));
2972	fInterceptMovDRx = true;
2973	}
2974	else
2975	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
2976
2977	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
2978	if (fInterceptDB)
2979	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
2980	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2981	{
2982	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2983	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
2984	#endif
2985	}
2986
2987	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
2988	if (fInterceptMovDRx)
2989	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2990	else
2991	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2992
2993	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2994	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2995
2996	/* The guest's view of its DR7 is unblemished. */
2997	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2998
2999	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3000	return rc;
3001	}
3002
3003
3004	#ifdef DEBUG
3005	/**
3006	* Debug function to validate segment registers.
3007	*/
3008	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3009	{
3010	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3011	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3012	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3013	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3014	{
3015	/** @todo DPL checks for CS, SS. */
3016	/* Protected mode checks */
3017	/* CS */
3018	Assert(pCtx->cs.Attr.n.u1Present);
3019	Assert(!(pCtx->cs.Attr.u & 0xf00));
3020	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3021	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3022	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3023	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3024	\|\| (pCtx->cs.Attr.n.u1Granularity));
3025	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3026	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3027	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3028	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3029	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3030	else
3031	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3032	/* SS */
3033	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3034	{
3035	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3036	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3037	Assert(pCtx->ss.Attr.n.u1Present);
3038	Assert(!(pCtx->ss.Attr.u & 0xf00));
3039	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3040	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3041	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3042	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3043	\|\| (pCtx->ss.Attr.n.u1Granularity));
3044	}
3045	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3046	/* CR0 might not be up-to-date here always, hence disabled. */
3047	#if 0
3048	if (!pCtx->cr0 & X86_CR0_PE)
3049	Assert(!pCtx->ss.Attr.n.u2Dpl);
3050	#endif
3051	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3052	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3053	{
3054	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3055	Assert(pCtx->ds.Attr.n.u1Present);
3056	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3057	Assert(!(pCtx->ds.Attr.u & 0xf00));
3058	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3059	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3060	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3061	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3062	\|\| (pCtx->ds.Attr.n.u1Granularity));
3063	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3064	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3065	}
3066	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3067	{
3068	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3069	Assert(pCtx->es.Attr.n.u1Present);
3070	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3071	Assert(!(pCtx->es.Attr.u & 0xf00));
3072	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3073	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3074	\|\| !(pCtx->es.Attr.n.u1Granularity));
3075	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3076	\|\| (pCtx->es.Attr.n.u1Granularity));
3077	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3078	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3079	}
3080	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3081	{
3082	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3083	Assert(pCtx->fs.Attr.n.u1Present);
3084	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3085	Assert(!(pCtx->fs.Attr.u & 0xf00));
3086	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3087	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3088	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3089	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3090	\|\| (pCtx->fs.Attr.n.u1Granularity));
3091	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3092	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3093	}
3094	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3095	{
3096	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3097	Assert(pCtx->gs.Attr.n.u1Present);
3098	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3099	Assert(!(pCtx->gs.Attr.u & 0xf00));
3100	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3101	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3102	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3103	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3104	\|\| (pCtx->gs.Attr.n.u1Granularity));
3105	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3106	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3107	}
3108	/* 64-bit capable CPUs. */
3109	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3110	Assert(!(pCtx->cs.u64Base >> 32));
3111	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3112	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3113	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3114	# endif
3115	}
3116	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3117	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3118	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3119	{
3120	/* Real and v86 mode checks. */
3121	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3122	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3123	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3124	{
3125	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3126	}
3127	else
3128	{
3129	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3130	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3131	}
3132
3133	/* CS */
3134	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3135	Assert(pCtx->cs.u32Limit == 0xffff);
3136	Assert(u32CSAttr == 0xf3);
3137	/* SS */
3138	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3139	Assert(pCtx->ss.u32Limit == 0xffff);
3140	Assert(u32SSAttr == 0xf3);
3141	/* DS */
3142	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3143	Assert(pCtx->ds.u32Limit == 0xffff);
3144	Assert(u32DSAttr == 0xf3);
3145	/* ES */
3146	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3147	Assert(pCtx->es.u32Limit == 0xffff);
3148	Assert(u32ESAttr == 0xf3);
3149	/* FS */
3150	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3151	Assert(pCtx->fs.u32Limit == 0xffff);
3152	Assert(u32FSAttr == 0xf3);
3153	/* GS */
3154	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3155	Assert(pCtx->gs.u32Limit == 0xffff);
3156	Assert(u32GSAttr == 0xf3);
3157	/* 64-bit capable CPUs. */
3158	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3159	Assert(!(pCtx->cs.u64Base >> 32));
3160	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3161	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3162	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3163	# endif
3164	}
3165	}
3166	#endif /* DEBUG */
3167
3168
3169	/**
3170	* Writes a guest segment register into the guest-state area in the VMCS.
3171	*
3172	* @returns VBox status code.
3173	* @param pVM Pointer to the VM.
3174	* @param pVCpu Pointer to the VMCPU.
3175	* @param idxSel Index of the selector in the VMCS.
3176	* @param idxLimit Index of the segment limit in the VMCS.
3177	* @param idxBase Index of the segment base in the VMCS.
3178	* @param idxAccess Index of the access rights of the segment in the VMCS.
3179	* @param pSelReg Pointer to the segment selector.
3180	* @param pCtx Pointer to the guest-CPU context.
3181	*
3182	* @remarks No-long-jump zone!!!
3183	*/
3184	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3185	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3186	{
3187	int rc;
3188	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3189	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3190	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3191	AssertRCReturn(rc, rc);
3192
3193	uint32_t u32Access = pSelReg->Attr.u;
3194	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3195	{
3196	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3197	u32Access = 0xf3;
3198	Assert(pVM->hm.s.vmx.pRealModeTSS);
3199	Assert(PDMVmmDevHeapIsEnabled(pVM));
3200	}
3201	else
3202	{
3203	/*
3204	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3205	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3206	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3207	* loaded in protected-mode have their attribute as 0.
3208	*/
3209	if (!u32Access)
3210	u32Access = VMX_SEL_UNUSABLE;
3211	}
3212
3213	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3214	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3215	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3216
3217	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3218	AssertRCReturn(rc, rc);
3219	return rc;
3220	}
3221
3222
3223	/**
3224	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3225	* into the guest-state area in the VMCS.
3226	*
3227	* @returns VBox status code.
3228	* @param pVM Pointer to the VM.
3229	* @param pVCPU Pointer to the VMCPU.
3230	* @param pCtx Pointer to the guest-CPU context.
3231	*
3232	* @remarks No-long-jump zone!!!
3233	* @remarks Requires RFLAGS (for debug assertions).
3234	*/
3235	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3236	{
3237	int rc = VERR_INTERNAL_ERROR_5;
3238
3239	/*
3240	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3241	*/
3242	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3243	{
3244	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3245	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3246	{
3247	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3248	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3249	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3250	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3251	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3252	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3253	}
3254
3255	#ifdef VBOX_WITH_REM
3256	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3257	{
3258	Assert(pVM->hm.s.vmx.pRealModeTSS);
3259	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3260	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3261	{
3262	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3263	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3264	&& enmGuestMode >= PGMMODE_PROTECTED)
3265	{
3266	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3267	in real-mode (e.g. OpenBSD 4.0) */
3268	REMFlushTBs(pVM);
3269	Log2(("Switch to protected mode detected!\n"));
3270	}
3271	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3272	}
3273	}
3274	#endif
3275	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3276	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3277	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3278	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3279	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3280	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3281	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3282	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3283	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3284	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3285	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3286	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3287	AssertRCReturn(rc, rc);
3288
3289	#ifdef DEBUG
3290	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3291	#endif
3292	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3293	}
3294
3295	/*
3296	* Guest TR.
3297	*/
3298	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3299	{
3300	/*
3301	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3302	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3303	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3304	*/
3305	uint16_t u16Sel = 0;
3306	uint32_t u32Limit = 0;
3307	uint64_t u64Base = 0;
3308	uint32_t u32AccessRights = 0;
3309
3310	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3311	{
3312	u16Sel = pCtx->tr.Sel;
3313	u32Limit = pCtx->tr.u32Limit;
3314	u64Base = pCtx->tr.u64Base;
3315	u32AccessRights = pCtx->tr.Attr.u;
3316	}
3317	else
3318	{
3319	Assert(pVM->hm.s.vmx.pRealModeTSS);
3320	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3321
3322	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3323	RTGCPHYS GCPhys;
3324	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3325	AssertRCReturn(rc, rc);
3326
3327	X86DESCATTR DescAttr;
3328	DescAttr.u = 0;
3329	DescAttr.n.u1Present = 1;
3330	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3331
3332	u16Sel = 0;
3333	u32Limit = HM_VTX_TSS_SIZE;
3334	u64Base = GCPhys; /* in real-mode phys = virt. */
3335	u32AccessRights = DescAttr.u;
3336	}
3337
3338	/* Validate. */
3339	Assert(!(u16Sel & RT_BIT(2)));
3340	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3341	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3342	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3343	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3344	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3345	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3346	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3347	Assert( (u32Limit & 0xfff) == 0xfff
3348	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3349	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3350	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3351
3352	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3353	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3354	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3355	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3356	AssertRCReturn(rc, rc);
3357
3358	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3359	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3360	}
3361
3362	/*
3363	* Guest GDTR.
3364	*/
3365	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3366	{
3367	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3368	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3369	AssertRCReturn(rc, rc);
3370
3371	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3372	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3373	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3374	}
3375
3376	/*
3377	* Guest LDTR.
3378	*/
3379	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3380	{
3381	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3382	uint32_t u32Access = 0;
3383	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3384	u32Access = VMX_SEL_UNUSABLE;
3385	else
3386	u32Access = pCtx->ldtr.Attr.u;
3387
3388	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3389	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3390	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3391	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3392	AssertRCReturn(rc, rc);
3393
3394	/* Validate. */
3395	if (!(u32Access & VMX_SEL_UNUSABLE))
3396	{
3397	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3398	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3399	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3400	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3401	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3402	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3403	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3404	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3405	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3406	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3407	}
3408
3409	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3410	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3411	}
3412
3413	/*
3414	* Guest IDTR.
3415	*/
3416	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3417	{
3418	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3419	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3420	AssertRCReturn(rc, rc);
3421
3422	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3423	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3424	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3425	}
3426
3427	/*
3428	* Guest FS & GS base MSRs.
3429	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3430	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3431	* are only available in 64-bit mode.
3432	*/
3433	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3434	* should not be necessary to do it in assembly again. */
3435	if (CPUMIsGuestInLongModeEx(pCtx))
3436	{
3437	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3438	{
3439	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3440	AssertRCReturn(rc, rc);
3441	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3442	}
3443
3444	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3445	{
3446	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3447	AssertRCReturn(rc, rc);
3448	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3449	}
3450	}
3451	else
3452	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3453
3454	return VINF_SUCCESS;
3455	}
3456
3457
3458	/**
3459	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3460	* areas. These MSRs will automatically be loaded to the host CPU on every
3461	* successful VM entry and stored from the host CPU on every successful VM exit.
3462	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3463	*
3464	* @returns VBox status code.
3465	* @param pVM Pointer to the VM.
3466	* @param pVCpu Pointer to the VMCPU.
3467	* @param pCtx Pointer to the guest-CPU context.
3468	*
3469	* @remarks No-long-jump zone!!!
3470	*/
3471	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3472	{
3473	AssertPtr(pVCpu);
3474	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3475
3476	/*
3477	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3478	*/
3479	int rc = VINF_SUCCESS;
3480	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3481	{
3482	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3483	unsigned cGuestMsrs = 0;
3484
3485	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3486	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3487	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3488	if (fSupportsNX \|\| fSupportsLongMode)
3489	{
3490	/** @todo support save IA32_EFER, i.e.
3491	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3492	* guest EFER need not be part of the VM-entry MSR-load area. */
3493	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3494	pGuestMsr->u32Reserved = 0;
3495	pGuestMsr->u64Value = pCtx->msrEFER;
3496	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3497	if (!CPUMIsGuestInLongModeEx(pCtx))
3498	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3499	pGuestMsr++; cGuestMsrs++;
3500	if (fSupportsLongMode)
3501	{
3502	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3503	pGuestMsr->u32Reserved = 0;
3504	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3505	pGuestMsr++; cGuestMsrs++;
3506	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3507	pGuestMsr->u32Reserved = 0;
3508	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3509	pGuestMsr++; cGuestMsrs++;
3510	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3511	pGuestMsr->u32Reserved = 0;
3512	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3513	pGuestMsr++; cGuestMsrs++;
3514	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3515	#if 0
3516	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3517	pGuestMsr->u32Reserved = 0;
3518	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3519	pGuestMsr++; cGuestMsrs++;
3520	#endif
3521	}
3522	}
3523
3524	/*
3525	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3526	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3527	*/
3528	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3529	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3530	{
3531	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3532	pGuestMsr->u32Reserved = 0;
3533	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3534	AssertRCReturn(rc, rc);
3535	pGuestMsr++; cGuestMsrs++;
3536	}
3537
3538	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3539	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3540	{
3541	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3542	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3543	}
3544
3545	/* Update the VCPU's copy of the guest MSR count. */
3546	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3547	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3548	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3549	AssertRCReturn(rc, rc);
3550
3551	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3552	}
3553
3554	/*
3555	* Guest Sysenter MSRs.
3556	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3557	* VM exits on WRMSRs for these MSRs.
3558	*/
3559	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3560	{
3561	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3562	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3563	}
3564	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3565	{
3566	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3567	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3568	}
3569	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3570	{
3571	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3572	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3573	}
3574	AssertRCReturn(rc, rc);
3575
3576	return rc;
3577	}
3578
3579
3580	/**
3581	* Loads the guest activity state into the guest-state area in the VMCS.
3582	*
3583	* @returns VBox status code.
3584	* @param pVM Pointer to the VM.
3585	* @param pVCpu Pointer to the VMCPU.
3586	* @param pCtx Pointer to the guest-CPU context.
3587	*
3588	* @remarks No-long-jump zone!!!
3589	*/
3590	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3591	{
3592	/** @todo See if we can make use of other states, e.g.
3593	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3594	int rc = VINF_SUCCESS;
3595	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3596	{
3597	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3598	AssertRCReturn(rc, rc);
3599	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3600	}
3601	return rc;
3602	}
3603
3604
3605	/**
3606	* Loads the guest-interruptibility state (or "interrupt shadow" as AMD calls
3607	* it) into the guest-state area in the VMCS.
3608	*
3609	* @returns VBox status code.
3610	* @param pVM Pointer to the VM.
3611	* @param pVCpu Pointer to the VMCPU.
3612	* @param pCtx Pointer to the guest-CPU context.
3613	*
3614	* @remarks No-long-jump zone!!!
3615	* @remarks Requires RIP, RFLAGS.
3616	*/
3617	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3618	{
3619	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_INTR_STATE))
3620	return VINF_SUCCESS;
3621
3622	/*
3623	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3624	* inhibit interrupts or clear any existing interrupt-inhibition.
3625	*/
3626	uint32_t uIntrState = 0;
3627	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3628	{
3629	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3630	{
3631	/*
3632	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in VT-x
3633	* the flag's condition to be cleared is met and thus the cleared state is correct. Additionally, this means
3634	* we need not re-read the VMCS field on the VM-exit path and clear/set this flag on every VM-exit. Finally,
3635	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
3636	*/
3637	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3638	uIntrState = 0; /* Clear interrupt inhibition. */
3639	}
3640	else if (pCtx->eflags.u32 & X86_EFL_IF)
3641	{
3642	/** @todo Pretty sure we don't need to check for Rflags.IF here.
3643	* Interrupt-shadow only matters when RIP changes. */
3644	/*
3645	* We don't have enough information to distinguish a block-by-STI vs. block-by-MOV SS. Intel seems to think there
3646	* is a slight difference regarding MOV SS additionally blocking some debug exceptions.
3647	* See Intel spec. 24.2.2 "Guest Non-Register State" table "Format of Interruptibility State".
3648	*/
3649	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3650	}
3651	}
3652	else
3653	uIntrState = 0; /* No interrupt inhibition. */
3654
3655	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3656	Assert((pCtx->eflags.u32 & X86_EFL_IF) \|\| uIntrState == 0); /* If EFLAGS.IF is not set, no interrupt inhibition. */
3657	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3658	AssertRCReturn(rc ,rc);
3659	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_INTR_STATE;
3660	return rc;
3661	}
3662
3663
3664	/**
3665	* Sets up the appropriate function to run guest code.
3666	*
3667	* @returns VBox status code.
3668	* @param pVM Pointer to the VM.
3669	* @param pVCpu Pointer to the VMCPU.
3670	* @param pCtx Pointer to the guest-CPU context.
3671	*
3672	* @remarks No-long-jump zone!!!
3673	* @todo change this to return void.
3674	*/
3675	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3676	{
3677	if (CPUMIsGuestInLongModeEx(pCtx))
3678	{
3679	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3680	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3681	#endif
3682	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3683	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3684	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3685	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3686	#else
3687	/* 64-bit host or hybrid host. */
3688	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3689	#endif
3690	}
3691	else
3692	{
3693	/* Guest is not in long mode, use the 32-bit handler. */
3694	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3695	}
3696	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3697	return VINF_SUCCESS;
3698	}
3699
3700
3701	/**
3702	* Wrapper for running the guest code in VT-x.
3703	*
3704	* @returns VBox strict status code.
3705	* @param pVM Pointer to the VM.
3706	* @param pVCpu Pointer to the VMCPU.
3707	* @param pCtx Pointer to the guest-CPU context.
3708	*
3709	* @remarks No-long-jump zone!!!
3710	*/
3711	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3712	{
3713	/*
3714	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3715	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3716	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3717	*/
3718	#ifdef VBOX_WITH_KERNEL_USING_XMM
3719	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3720	#else
3721	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3722	#endif
3723	}
3724
3725
3726	/**
3727	* Report world-switch error and dump some useful debug info.
3728	*
3729	* @param pVM Pointer to the VM.
3730	* @param pVCpu Pointer to the VMCPU.
3731	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3732	* @param pCtx Pointer to the guest-CPU context.
3733	* @param pVmxTransient Pointer to the VMX transient structure (only
3734	* exitReason updated).
3735	*/
3736	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3737	{
3738	Assert(pVM);
3739	Assert(pVCpu);
3740	Assert(pCtx);
3741	Assert(pVmxTransient);
3742	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3743
3744	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3745	switch (rcVMRun)
3746	{
3747	case VERR_VMX_INVALID_VMXON_PTR:
3748	AssertFailed();
3749	break;
3750	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3751	case VERR_VMX_UNABLE_TO_START_VM:
3752	case VERR_VMX_UNABLE_TO_RESUME_VM:
3753	{
3754	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3755	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3756	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3757	AssertRC(rc);
3758
3759	#ifdef VBOX_STRICT
3760	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3761	pVmxTransient->uExitReason));
3762	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3763	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3764	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3765	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3766	else
3767	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3768
3769	/* VMX control bits. */
3770	uint32_t u32Val;
3771	uint64_t u64Val;
3772	RTHCUINTREG uHCReg;
3773	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3774	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3775	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3776	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3777	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3778	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3779	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3780	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3781	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3782	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3783	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3784	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3785	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3786	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3787	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3788	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3789	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3790	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3791	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3792	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3793	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3794	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3796	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3798	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3800	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3802	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3804	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3805	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3806	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3807	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3808	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3809	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3810	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3811	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3812	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3813	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3814	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3815
3816	/* Guest bits. */
3817	RTGCUINTREG uGCReg;
3818	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3819	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3820	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3821	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3822	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3823	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3824	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3826
3827	/* Host bits. */
3828	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3829	Log(("Host CR0 %#RHr\n", uHCReg));
3830	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3831	Log(("Host CR3 %#RHr\n", uHCReg));
3832	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3833	Log(("Host CR4 %#RHr\n", uHCReg));
3834
3835	RTGDTR HostGdtr;
3836	PCX86DESCHC pDesc;
3837	ASMGetGDTR(&HostGdtr);
3838	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3839	Log(("Host CS %#08x\n", u32Val));
3840	if (u32Val < HostGdtr.cbGdt)
3841	{
3842	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3843	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3844	}
3845
3846	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3847	Log(("Host DS %#08x\n", u32Val));
3848	if (u32Val < HostGdtr.cbGdt)
3849	{
3850	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3851	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3852	}
3853
3854	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3855	Log(("Host ES %#08x\n", u32Val));
3856	if (u32Val < HostGdtr.cbGdt)
3857	{
3858	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3859	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3860	}
3861
3862	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3863	Log(("Host FS %#08x\n", u32Val));
3864	if (u32Val < HostGdtr.cbGdt)
3865	{
3866	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3867	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3868	}
3869
3870	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3871	Log(("Host GS %#08x\n", u32Val));
3872	if (u32Val < HostGdtr.cbGdt)
3873	{
3874	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3875	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3876	}
3877
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3879	Log(("Host SS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3887	Log(("Host TR %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3892	}
3893
3894	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3895	Log(("Host TR Base %#RHv\n", uHCReg));
3896	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3897	Log(("Host GDTR Base %#RHv\n", uHCReg));
3898	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3899	Log(("Host IDTR Base %#RHv\n", uHCReg));
3900	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3901	Log(("Host SYSENTER CS %#08x\n", u32Val));
3902	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3903	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3904	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3905	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3906	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3907	Log(("Host RSP %#RHv\n", uHCReg));
3908	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3909	Log(("Host RIP %#RHv\n", uHCReg));
3910	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3911	if (VMX_IS_64BIT_HOST_MODE())
3912	{
3913	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3914	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3915	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3916	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3917	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3918	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3919	}
3920	# endif
3921	#endif /* VBOX_STRICT */
3922	break;
3923	}
3924
3925	default:
3926	/* Impossible */
3927	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3928	break;
3929	}
3930	NOREF(pVM);
3931	}
3932
3933
3934	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3935	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3936	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3937	#endif
3938
3939	/**
3940	* Executes the specified handler in 64-bit mode.
3941	*
3942	* @returns VBox status code.
3943	* @param pVM Pointer to the VM.
3944	* @param pVCpu Pointer to the VMCPU.
3945	* @param pCtx Pointer to the guest CPU context.
3946	* @param pfnHandler Pointer to the RC handler function.
3947	* @param cbParam Number of parameters.
3948	* @param paParam Array of 32-bit parameters.
3949	*/
3950	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3951	uint32_t *paParam)
3952	{
3953	int rc, rc2;
3954	PHMGLOBLCPUINFO pCpu;
3955	RTHCPHYS HCPhysCpuPage;
3956	RTHCUINTREG uOldEFlags;
3957
3958	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3959	Assert(pfnHandler);
3960	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3961	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3962
3963	#ifdef VBOX_STRICT
3964	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3965	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3966
3967	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3968	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3969	#endif
3970
3971	/* Disable interrupts. */
3972	uOldEFlags = ASMIntDisableFlags();
3973
3974	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3975	RTCPUID idHostCpu = RTMpCpuId();
3976	CPUMR0SetLApic(pVM, idHostCpu);
3977	#endif
3978
3979	pCpu = HMR0GetCurrentCpu();
3980	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
3981
3982	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3983	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
3984
3985	/* Leave VMX Root Mode. */
3986	VMXDisable();
3987
3988	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3989
3990	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3991	CPUMSetHyperEIP(pVCpu, pfnHandler);
3992	for (int i = (int)cbParam - 1; i >= 0; i--)
3993	CPUMPushHyper(pVCpu, paParam[i]);
3994
3995	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3996
3997	/* Call the switcher. */
3998	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
3999	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4000
4001	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4002	/* Make sure the VMX instructions don't cause #UD faults. */
4003	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4004
4005	/* Re-enter VMX Root Mode */
4006	rc2 = VMXEnable(HCPhysCpuPage);
4007	if (RT_FAILURE(rc2))
4008	{
4009	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4010	ASMSetFlags(uOldEFlags);
4011	return VERR_VMX_VMXON_FAILED;
4012	}
4013
4014	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4015	AssertRC(rc2);
4016	Assert(!(ASMGetFlags() & X86_EFL_IF));
4017	ASMSetFlags(uOldEFlags);
4018	return rc;
4019	}
4020
4021
4022	/**
4023	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4024	* supporting 64-bit guests.
4025	*
4026	* @returns VBox status code.
4027	* @param fResume Whether to VMLAUNCH or VMRESUME.
4028	* @param pCtx Pointer to the guest-CPU context.
4029	* @param pCache Pointer to the VMCS cache.
4030	* @param pVM Pointer to the VM.
4031	* @param pVCpu Pointer to the VMCPU.
4032	*/
4033	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4034	{
4035	uint32_t aParam[6];
4036	PHMGLOBLCPUINFO pCpu = NULL;
4037	RTHCPHYS HCPhysCpuPage = 0;
4038	int rc = VERR_INTERNAL_ERROR_5;
4039
4040	pCpu = HMR0GetCurrentCpu();
4041	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4042
4043	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4044	pCache->uPos = 1;
4045	pCache->interPD = PGMGetInterPaeCR3(pVM);
4046	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4047	#endif
4048
4049	#ifdef DEBUG
4050	pCache->TestIn.HCPhysCpuPage= 0;
4051	pCache->TestIn.HCPhysVmcs = 0;
4052	pCache->TestIn.pCache = 0;
4053	pCache->TestOut.HCPhysVmcs = 0;
4054	pCache->TestOut.pCache = 0;
4055	pCache->TestOut.pCtx = 0;
4056	pCache->TestOut.eflags = 0;
4057	#endif
4058
4059	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4060	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4061	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4062	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4063	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4064	aParam[5] = 0;
4065
4066	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4067	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4068	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4069	#endif
4070	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4071
4072	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4073	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4074	Assert(pCtx->dr[4] == 10);
4075	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4076	#endif
4077
4078	#ifdef DEBUG
4079	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4080	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4081	pVCpu->hm.s.vmx.HCPhysVmcs));
4082	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4083	pCache->TestOut.HCPhysVmcs));
4084	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4085	pCache->TestOut.pCache));
4086	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4087	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4088	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4089	pCache->TestOut.pCtx));
4090	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4091	#endif
4092	return rc;
4093	}
4094
4095
4096	/**
4097	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4098	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4099	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4100	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4101	*
4102	* @returns VBox status code.
4103	* @param pVM Pointer to the VM.
4104	* @param pVCpu Pointer to the VMCPU.
4105	*/
4106	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4107	{
4108	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4109	{ \
4110	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4111	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4112	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4113	}
4114
4115	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4116	{ \
4117	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4118	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4119	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4120	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4121	}
4122
4123	AssertPtr(pVM);
4124	AssertPtr(pVCpu);
4125	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4126
4127	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4128	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4129	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4130	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4131	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4132	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4133	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4134	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4135	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4136
4137	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4138	#if 0
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4142	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4143	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4144	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4145	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4146	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4147	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4148	#endif
4149
4150	/* 32-bit guest-state fields. */
4151	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4152	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4153	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4154	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4155	/* Unused 32-bit guest-state fields. */
4156	#if 0
4157	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4158	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4159	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4160	#endif
4161
4162	/* Natural width guest-state fields. */
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4166	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4167	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4169	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4170	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4171	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4172	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4173	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4174	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4175	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4176	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4177	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4178	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4179	/* Unused natural width guest-state fields. */
4180	#if 0
4181	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4182	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4183	#endif
4184	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4185	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4186
4187	if (pVM->hm.s.fNestedPaging)
4188	{
4189	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4190	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4191	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4192	}
4193	else
4194	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4195
4196	#undef VMXLOCAL_INIT_VMCS_SELREG
4197	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4198	return VINF_SUCCESS;
4199	}
4200
4201
4202	/**
4203	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4204	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4205	* darwin, running 64-bit guests).
4206	*
4207	* @returns VBox status code.
4208	* @param pVCpu Pointer to the VMCPU.
4209	* @param idxField The VMCS field encoding.
4210	* @param u64Val 16, 32 or 64 bits value.
4211	*/
4212	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4213	{
4214	int rc;
4215	switch (idxField)
4216	{
4217	/*
4218	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4219	*/
4220	/* 64-bit Control fields. */
4221	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4222	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4223	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4224	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4225	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4226	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4227	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4228	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4229	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4230	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4231	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4232	case VMX_VMCS64_CTRL_EPTP_FULL:
4233	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4234	/* 64-bit Read-only data fields. */
4235	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4236	/* 64-bit Guest-state fields. */
4237	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4238	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4239	case VMX_VMCS64_GUEST_PAT_FULL:
4240	case VMX_VMCS64_GUEST_EFER_FULL:
4241	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4242	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4243	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4244	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4245	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4246	/* 64-bit Host-state fields. */
4247	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4248	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4249	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4250	{
4251	rc = VMXWriteVmcs32(idxField, u64Val);
4252	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4253	break;
4254	}
4255
4256	/*
4257	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4258	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4259	*/
4260	/* Natural-width Guest-state fields. */
4261	case VMX_VMCS_GUEST_CR0:
4262	case VMX_VMCS_GUEST_CR3:
4263	case VMX_VMCS_GUEST_CR4:
4264	case VMX_VMCS_GUEST_ES_BASE:
4265	case VMX_VMCS_GUEST_CS_BASE:
4266	case VMX_VMCS_GUEST_SS_BASE:
4267	case VMX_VMCS_GUEST_DS_BASE:
4268	case VMX_VMCS_GUEST_FS_BASE:
4269	case VMX_VMCS_GUEST_GS_BASE:
4270	case VMX_VMCS_GUEST_LDTR_BASE:
4271	case VMX_VMCS_GUEST_TR_BASE:
4272	case VMX_VMCS_GUEST_GDTR_BASE:
4273	case VMX_VMCS_GUEST_IDTR_BASE:
4274	case VMX_VMCS_GUEST_DR7:
4275	case VMX_VMCS_GUEST_RSP:
4276	case VMX_VMCS_GUEST_RIP:
4277	case VMX_VMCS_GUEST_RFLAGS:
4278	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4279	case VMX_VMCS_GUEST_SYSENTER_ESP:
4280	case VMX_VMCS_GUEST_SYSENTER_EIP:
4281	{
4282	if ((u64Val >> 32ULL) == 0)
4283	{
4284	/* If this field is 64-bit, VT-x will zero out the top bits. */
4285	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4286	}
4287	else
4288	{
4289	/* Assert that only the 32->64 switcher case should ever come here. */
4290	Assert(pVM->hm.s.fAllow64BitGuests);
4291	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4292	}
4293	break;
4294	}
4295
4296	default:
4297	{
4298	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4299	rc = VERR_INVALID_PARAMETER;
4300	break;
4301	}
4302	}
4303	AssertRCReturn(rc, rc);
4304	return rc;
4305	}
4306
4307
4308	/**
4309	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4310	* hosts (except darwin) for 64-bit guests.
4311	*
4312	* @param pVCpu Pointer to the VMCPU.
4313	* @param idxField The VMCS field encoding.
4314	* @param u64Val 16, 32 or 64 bits value.
4315	*/
4316	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4317	{
4318	AssertPtr(pVCpu);
4319	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4320
4321	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4322	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4323
4324	/* Make sure there are no duplicates. */
4325	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4326	{
4327	if (pCache->Write.aField[i] == idxField)
4328	{
4329	pCache->Write.aFieldVal[i] = u64Val;
4330	return VINF_SUCCESS;
4331	}
4332	}
4333
4334	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4335	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4336	pCache->Write.cValidEntries++;
4337	return VINF_SUCCESS;
4338	}
4339
4340
4341	/**
4342	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4343	*
4344	* @param pCache Pointer to the VMCS cache.
4345	*/
4346	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4347	{
4348	AssertPtr(pCache);
4349	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4350	{
4351	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4352	AssertRC(rc, rc);
4353	}
4354	pCache->Write.cValidEntries = 0;
4355	}
4356
4357
4358	/**
4359	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4360	*
4361	* @param pCache Pointer to the VMCS cache.
4362	* @remarks No-long-jump zone!!!
4363	*/
4364	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4365	{
4366	AssertPtr(pCache);
4367	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4368	{
4369	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4370	AssertRC(rc);
4371	}
4372	}
4373	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4374
4375
4376	/**
4377	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4378	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4379	* timer.
4380	*
4381	* @returns VBox status code.
4382	* @param pVM Pointer to the VM.
4383	* @param pVCpu Pointer to the VMCPU.
4384	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4385	* out-of-sync. Make sure to update the required fields
4386	* before using them.
4387	* @remarks No-long-jump zone!!!
4388	*/
4389	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4390	{
4391	int rc = VERR_INTERNAL_ERROR_5;
4392	bool fOffsettedTsc = false;
4393	if (pVM->hm.s.vmx.fUsePreemptTimer)
4394	{
4395	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4396
4397	/* Make sure the returned values have sane upper and lower boundaries. */
4398	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4399	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4400	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4401	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4402
4403	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4404	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4405	}
4406	else
4407	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4408
4409	if (fOffsettedTsc)
4410	{
4411	uint64_t u64CurTSC = ASMReadTSC();
4412	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4413	{
4414	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4415	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4416
4417	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4418	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4419	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4420	}
4421	else
4422	{
4423	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4424	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4425	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4426	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4427	}
4428	}
4429	else
4430	{
4431	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4432	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4433	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4434	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4435	}
4436	}
4437
4438
4439	/**
4440	* Determines if an exception is a benign exception. Benign exceptions
4441	* are ones which cannot cause double-faults.
4442	*
4443	* @returns true if the exception is benign, false otherwise.
4444	* @param uVector The exception vector.
4445	*/
4446	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint8_t uVector)
4447	{
4448	switch (uVector)
4449	{
4450	case X86_XCPT_DB:
4451	case X86_XCPT_NMI:
4452	case X86_XCPT_BP:
4453	case X86_XCPT_OF:
4454	case X86_XCPT_BR:
4455	case X86_XCPT_UD:
4456	case X86_XCPT_NM:
4457	case X86_XCPT_CO_SEG_OVERRUN:
4458	case X86_XCPT_MF:
4459	case X86_XCPT_AC:
4460	case X86_XCPT_MC:
4461	case X86_XCPT_XF:
4462	return true;
4463	default:
4464	return false;
4465	}
4466	}
4467
4468
4469	/**
4470	* Determines if an exception is a contributory exception. Contributory
4471	* exceptions are ones which can cause double-faults.
4472	*
4473	* @returns true if the exception is contributory, false otherwise.
4474	* @param uVector The exception vector.
4475	*/
4476	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint8_t uVector)
4477	{
4478	switch (uVector)
4479	{
4480	case X86_XCPT_GP:
4481	case X86_XCPT_SS:
4482	case X86_XCPT_NP:
4483	case X86_XCPT_TS:
4484	case X86_XCPT_DE:
4485	return true;
4486	default:
4487	return false;
4488	}
4489	return false;
4490	}
4491
4492
4493	/**
4494	* Determines if we are intercepting any contributory exceptions.
4495	*
4496	* @returns true if we are intercepting any contributory exception, false
4497	* otherwise.
4498	* @param pVCpu Pointer to the VMCPU.
4499	*/
4500	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4501	{
4502	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4503	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4504	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4505	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4506	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4507	{
4508	return true;
4509	}
4510	return false;
4511	}
4512
4513
4514	/**
4515	* Handle a condition that occurred while delivering an event through the guest
4516	* IDT.
4517	*
4518	* @returns VBox status code (informational error codes included).
4519	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4520	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4521	* continue execution of the guest which will delivery the #DF.
4522	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4523	*
4524	* @param pVM Pointer to the VM.
4525	* @param pVCpu Pointer to the VMCPU.
4526	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4527	* out-of-sync. Make sure to update the required fields
4528	* before using them.
4529	* @param pVmxTransient Pointer to the VMX transient structure.
4530	*
4531	* @remarks No-long-jump zone!!!
4532	*/
4533	static int hmR0VmxCheckExitDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4534	{
4535	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4536	AssertRCReturn(rc, rc);
4537	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4538	{
4539	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4540	AssertRCReturn(rc, rc);
4541
4542	uint8_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4543	uint8_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4544	uint8_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4545
4546	typedef enum
4547	{
4548	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4549	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4550	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4551	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4552	} VMXREFLECTXCPT;
4553
4554	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4555	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4556	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4557	{
4558	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4559	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4560	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4561	&& uExitVector == X86_XCPT_PF))
4562	{
4563	enmReflect = VMXREFLECTXCPT_XCPT;
4564	}
4565	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4566	&& hmR0VmxIsContributoryXcpt(uExitVector))
4567	{
4568	enmReflect = VMXREFLECTXCPT_DF;
4569	}
4570	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4571	&& uIdtVector == X86_XCPT_PF
4572	&& uExitVector == X86_XCPT_PF)
4573	{
4574	pVmxTransient->fVectoringPF = true;
4575	}
4576	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4577	&& uIdtVector == X86_XCPT_PF
4578	&& hmR0VmxIsContributoryXcpt(uExitVector))
4579	{
4580	enmReflect = VMXREFLECTXCPT_DF;
4581	}
4582	else if (uIdtVector == X86_XCPT_DF)
4583	enmReflect = VMXREFLECTXCPT_TF;
4584	}
4585	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4586	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4587	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4588	{
4589	/*
4590	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4591	* (whatever they are) as they reoccur when restarting the instruction.
4592	*/
4593	enmReflect = VMXREFLECTXCPT_XCPT;
4594	}
4595
4596	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4597	switch (enmReflect)
4598	{
4599	case VMXREFLECTXCPT_XCPT:
4600	{
4601	Assert(!pVCpu->hm.s.Event.fPending);
4602	pVCpu->hm.s.Event.fPending = true;
4603	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4604	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4605	{
4606	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4607	AssertRCReturn(rc, rc);
4608	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4609	}
4610	else
4611	pVCpu->hm.s.Event.u32ErrCode = 0;
4612	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4613	break;
4614	}
4615
4616	case VMXREFLECTXCPT_DF:
4617	{
4618	uint32_t u32IntrInfo;
4619	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4620	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4621	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4622
4623	Assert(!pVCpu->hm.s.Event.fPending);
4624	pVCpu->hm.s.Event.fPending = true;
4625	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4626	pVCpu->hm.s.Event.u32ErrCode = 0;
4627	rc = VINF_VMX_DOUBLE_FAULT;
4628	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4629	break;
4630	}
4631
4632	case VMXREFLECTXCPT_TF:
4633	{
4634	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4635	rc = VINF_EM_RESET;
4636	break;
4637	}
4638
4639	default: /* shut up gcc. */
4640	break;
4641	}
4642	}
4643	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4644	return rc;
4645	}
4646
4647
4648	/**
4649	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4650	*
4651	* @returns VBox status code.
4652	* @param pVM Pointer to the VM.
4653	* @param pVCpu Pointer to the VMCPU.
4654	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4655	* out-of-sync. Make sure to update the required fields
4656	* before using them.
4657	*
4658	* @remarks No-long-jump zone!!!
4659	*/
4660	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4661	{
4662	int rc = VINF_SUCCESS;
4663	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4664	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4665	{
4666	RTGCUINTREG uVal = 0;
4667	RTCCUINTREG uShadow = 0;
4668	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4669	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4670	AssertRCReturn(rc, rc);
4671	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4672	CPUMSetGuestCR0(pVCpu, uVal);
4673	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4674	}
4675	return rc;
4676	}
4677
4678
4679	/**
4680	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4681	*
4682	* @returns VBox status code.
4683	* @param pVM Pointer to the VM.
4684	* @param pVCpu Pointer to the VMCPU.
4685	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4686	* out-of-sync. Make sure to update the required fields
4687	* before using them.
4688	*
4689	* @remarks No-long-jump zone!!!
4690	*/
4691	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4692	{
4693	int rc = VINF_SUCCESS;
4694	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4695	{
4696	RTGCUINTREG uVal = 0;
4697	RTCCUINTREG uShadow = 0;
4698	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4699	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4700	AssertRCReturn(rc, rc);
4701	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4702	CPUMSetGuestCR4(pVCpu, uVal);
4703	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4704	}
4705	return rc;
4706	}
4707
4708
4709	/**
4710	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4711	*
4712	* @returns VBox status code.
4713	* @param pVM Pointer to the VM.
4714	* @param pVCpu Pointer to the VMCPU.
4715	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4716	* out-of-sync. Make sure to update the required fields
4717	* before using them.
4718	*
4719	* @remarks No-long-jump zone!!!
4720	*/
4721	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4722	{
4723	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4724	return VINF_SUCCESS;
4725
4726	RTGCUINTREG uVal = 0;
4727	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4728	AssertRCReturn(rc, rc);
4729	pMixedCtx->rip = uVal;
4730	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4731	return rc;
4732	}
4733
4734
4735	/**
4736	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4737	*
4738	* @returns VBox status code.
4739	* @param pVM Pointer to the VM.
4740	* @param pVCpu Pointer to the VMCPU.
4741	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4742	* out-of-sync. Make sure to update the required fields
4743	* before using them.
4744	*
4745	* @remarks No-long-jump zone!!!
4746	*/
4747	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4748	{
4749	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4750	return VINF_SUCCESS;
4751
4752	RTGCUINTREG uVal = 0;
4753	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4754	AssertRCReturn(rc, rc);
4755	pMixedCtx->rsp = uVal;
4756	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4757	return rc;
4758	}
4759
4760
4761	/**
4762	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4763	*
4764	* @returns VBox status code.
4765	* @param pVM Pointer to the VM.
4766	* @param pVCpu Pointer to the VMCPU.
4767	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4768	* out-of-sync. Make sure to update the required fields
4769	* before using them.
4770	*
4771	* @remarks No-long-jump zone!!!
4772	*/
4773	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4774	{
4775	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4776	return VINF_SUCCESS;
4777
4778	RTGCUINTREG uVal = 0;
4779	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4780	AssertRCReturn(rc, rc);
4781	pMixedCtx->rflags.u64 = uVal;
4782
4783	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4784	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4785	{
4786	Assert(pVM->hm.s.vmx.pRealModeTSS);
4787	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4788	pMixedCtx->eflags.Bits.u1VM = 0;
4789	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4790	}
4791
4792	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4793	return rc;
4794	}
4795
4796
4797	/**
4798	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4799	* guest-CPU context.
4800	*/
4801	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4802	{
4803	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4804	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4805	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4806	AssertRCReturn(rc, rc);
4807	return rc;
4808	}
4809
4810
4811	/**
4812	* Saves the guest's interruptibility state.
4813	*
4814	* @returns VBox status code.
4815	* @param pVM Pointer to the VM.
4816	* @param pVCpu Pointer to the VMCPU.
4817	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4818	* out-of-sync. Make sure to update the required fields
4819	* before using them.
4820	*
4821	* @remarks No-long-jump zone!!!
4822	*/
4823	DECLINLINE(int) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4824	{
4825	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_INTR_STATE)
4826	return VINF_SUCCESS;
4827
4828	uint32_t uIntrState = 0;
4829	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4830	if (!uIntrState)
4831	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4832	else
4833	{
4834	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4835	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4836	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4837	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* RFLAGS is needed in hmR0VmxLoadGuestIntrState(). */
4838	AssertRCReturn(rc, rc);
4839	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4840	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4841	}
4842
4843	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_INTR_STATE;
4844	return rc;
4845	}
4846
4847
4848	/**
4849	* Saves the guest's activity state.
4850	*
4851	* @returns VBox status code.
4852	* @param pVM Pointer to the VM.
4853	* @param pVCpu Pointer to the VMCPU.
4854	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4855	* out-of-sync. Make sure to update the required fields
4856	* before using them.
4857	*
4858	* @remarks No-long-jump zone!!!
4859	*/
4860	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4861	{
4862	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4863	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4864	return VINF_SUCCESS;
4865	}
4866
4867
4868	/**
4869	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4870	* the current VMCS into the guest-CPU context.
4871	*
4872	* @returns VBox status code.
4873	* @param pVM Pointer to the VM.
4874	* @param pVCpu Pointer to the VMCPU.
4875	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4876	* out-of-sync. Make sure to update the required fields
4877	* before using them.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882	{
4883	int rc = VINF_SUCCESS;
4884	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4885	{
4886	uint32_t u32Val = 0;
4887	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4888	pMixedCtx->SysEnter.cs = u32Val;
4889	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4890	}
4891
4892	RTGCUINTREG uGCVal = 0;
4893	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4894	{
4895	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4896	pMixedCtx->SysEnter.eip = uGCVal;
4897	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4898	}
4899	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4900	{
4901	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4902	pMixedCtx->SysEnter.esp = uGCVal;
4903	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4904	}
4905	return rc;
4906	}
4907
4908
4909	/**
4910	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4911	* context.
4912	*
4913	* @returns VBox status code.
4914	* @param pVM Pointer to the VM.
4915	* @param pVCpu Pointer to the VMCPU.
4916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4917	* out-of-sync. Make sure to update the required fields
4918	* before using them.
4919	*
4920	* @remarks No-long-jump zone!!!
4921	*/
4922	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4923	{
4924	RTGCUINTREG uVal = 0;
4925	int rc = VINF_SUCCESS;
4926	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4927	{
4928	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4929	pMixedCtx->fs.u64Base = uVal;
4930	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4931	}
4932	return rc;
4933	}
4934
4935
4936	/**
4937	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4938	* context.
4939	*
4940	* @returns VBox status code.
4941	* @param pVM Pointer to the VM.
4942	* @param pVCpu Pointer to the VMCPU.
4943	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4944	* out-of-sync. Make sure to update the required fields
4945	* before using them.
4946	*
4947	* @remarks No-long-jump zone!!!
4948	*/
4949	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4950	{
4951	RTGCUINTREG uVal = 0;
4952	int rc = VINF_SUCCESS;
4953	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4954	{
4955	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4956	pMixedCtx->gs.u64Base = uVal;
4957	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4958	}
4959	return rc;
4960	}
4961
4962
4963	/**
4964	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4965	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4966	*
4967	* @returns VBox status code.
4968	* @param pVM Pointer to the VM.
4969	* @param pVCpu Pointer to the VMCPU.
4970	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4971	* out-of-sync. Make sure to update the required fields
4972	* before using them.
4973	*
4974	* @remarks No-long-jump zone!!!
4975	*/
4976	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4977	{
4978	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
4979	return VINF_SUCCESS;
4980
4981	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
4982	{
4983	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4984	pMsr += i;
4985	switch (pMsr->u32IndexMSR)
4986	{
4987	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
4988	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
4989	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
4990	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
4991	#if 0
4992	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
4993	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
4994	#endif
4995	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
4996	default:
4997	{
4998	AssertFailed();
4999	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5000	}
5001	}
5002	}
5003	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5004	return VINF_SUCCESS;
5005	}
5006
5007
5008	/**
5009	* Saves the guest control registers from the current VMCS into the guest-CPU
5010	* context.
5011	*
5012	* @returns VBox status code.
5013	* @param pVM Pointer to the VM.
5014	* @param pVCpu Pointer to the VMCPU.
5015	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5016	* out-of-sync. Make sure to update the required fields
5017	* before using them.
5018	*
5019	* @remarks No-long-jump zone!!!
5020	*/
5021	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5022	{
5023	RTGCUINTREG uVal = 0;
5024	RTGCUINTREG uShadow = 0;
5025	int rc = VINF_SUCCESS;
5026
5027	/* Guest CR0. Guest FPU. */
5028	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5029	AssertRCReturn(rc, rc);
5030
5031	/* Guest CR4. */
5032	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5033	AssertRCReturn(rc, rc);
5034
5035	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5036	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5037	{
5038	if ( pVM->hm.s.fNestedPaging
5039	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5040	{
5041	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5042	AssertRCReturn(rc, rc);
5043	if (pMixedCtx->cr3 != uVal)
5044	{
5045	CPUMSetGuestCR3(pVCpu, uVal);
5046	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5047	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5048	}
5049
5050	/* We require EFER to check PAE mode. */
5051	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5052	AssertRCReturn(rc, rc);
5053
5054	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5055	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5056	{
5057	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5058	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5059	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5060	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5061	AssertRCReturn(rc, rc);
5062	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5063	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5064	}
5065	}
5066	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5067	}
5068	return rc;
5069	}
5070
5071
5072	/**
5073	* Reads a guest segment register from the current VMCS into the guest-CPU
5074	* context.
5075	*
5076	* @returns VBox status code.
5077	* @param idxSel Index of the selector in the VMCS.
5078	* @param idxLimit Index of the segment limit in the VMCS.
5079	* @param idxBase Index of the segment base in the VMCS.
5080	* @param idxAccess Index of the access rights of the segment in the VMCS.
5081	* @param pSelReg Pointer to the segment selector.
5082	*
5083	* @remarks No-long-jump zone!!!
5084	*/
5085	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5086	PCPUMSELREG pSelReg)
5087	{
5088	uint32_t u32Val = 0;
5089	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5090	pSelReg->Sel = (uint16_t)u32Val;
5091	pSelReg->ValidSel = (uint16_t)u32Val;
5092	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5093
5094	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5095	pSelReg->u32Limit = u32Val;
5096
5097	RTGCUINTREG uGCVal = 0;
5098	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5099	pSelReg->u64Base = uGCVal;
5100
5101	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5102	pSelReg->Attr.u = u32Val;
5103
5104	/*
5105	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5106	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5107	*/
5108	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5109	{
5110	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5111	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5112	}
5113	return rc;
5114	}
5115
5116
5117	/**
5118	* Saves the guest segment registers from the current VMCS into the guest-CPU
5119	* context.
5120	*
5121	* @returns VBox status code.
5122	* @param pVM Pointer to the VM.
5123	* @param pVCpu Pointer to the VMCPU.
5124	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5125	* out-of-sync. Make sure to update the required fields
5126	* before using them.
5127	*
5128	* @remarks No-long-jump zone!!!
5129	*/
5130	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5131	{
5132	int rc = VINF_SUCCESS;
5133
5134	/* Guest segment registers. */
5135	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5136	{
5137	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5138	AssertRCReturn(rc, rc);
5139
5140	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5141	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5142	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5143	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5144	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5145	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5146	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5147	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5148	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5149	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5150	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5151	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5152	AssertRCReturn(rc, rc);
5153
5154	/* Restore segment attributes for real-on-v86 mode hack. */
5155	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5156	{
5157	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5158	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5159	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5160	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5161	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5162	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5163	}
5164
5165	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5166	}
5167
5168	/* Guest LDTR. */
5169	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5170	{
5171	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5172	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5173	AssertRCReturn(rc, rc);
5174	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5175	}
5176
5177	/* Guest GDTR. */
5178	RTGCUINTREG uGCVal = 0;
5179	uint32_t u32Val = 0;
5180	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5181	{
5182	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5183	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5184	pMixedCtx->gdtr.pGdt = uGCVal;
5185	pMixedCtx->gdtr.cbGdt = u32Val;
5186	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5187	}
5188
5189	/* Guest IDTR. */
5190	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5191	{
5192	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5193	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5194	pMixedCtx->idtr.pIdt = uGCVal;
5195	pMixedCtx->idtr.cbIdt = u32Val;
5196	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5197	}
5198
5199	/* Guest TR. */
5200	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5201	{
5202	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5203	AssertRCReturn(rc, rc);
5204
5205	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5206	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5207	{
5208	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5209	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5210	AssertRCReturn(rc, rc);
5211	}
5212	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5213	}
5214	return rc;
5215	}
5216
5217
5218	/**
5219	* Saves the guest debug registers from the current VMCS into the guest-CPU
5220	* context.
5221	*
5222	* @returns VBox status code.
5223	* @param pVM Pointer to the VM.
5224	* @param pVCpu Pointer to the VMCPU.
5225	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5226	* out-of-sync. Make sure to update the required fields
5227	* before using them.
5228	*
5229	* @remarks No-long-jump zone!!!
5230	*/
5231	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5232	{
5233	int rc = VINF_SUCCESS;
5234	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5235	{
5236	RTGCUINTREG uVal;
5237	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5238	pMixedCtx->dr[7] = uVal;
5239
5240	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5241	}
5242	return rc;
5243	}
5244
5245
5246	/**
5247	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5248	*
5249	* @returns VBox status code.
5250	* @param pVM Pointer to the VM.
5251	* @param pVCpu Pointer to the VMCPU.
5252	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5253	* out-of-sync. Make sure to update the required fields
5254	* before using them.
5255	*
5256	* @remarks No-long-jump zone!!!
5257	*/
5258	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5259	{
5260	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5261	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5262	return VINF_SUCCESS;
5263	}
5264
5265
5266	/**
5267	* Saves the entire guest state from the currently active VMCS into the
5268	* guest-CPU context. This essentially VMREADs all guest-data.
5269	*
5270	* @returns VBox status code.
5271	* @param pVM Pointer to the VM.
5272	* @param pVCpu Pointer to the VMCPU.
5273	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5274	* out-of-sync. Make sure to update the required fields
5275	* before using them.
5276	*/
5277	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5278	{
5279	Assert(pVM);
5280	Assert(pVCpu);
5281	Assert(pMixedCtx);
5282
5283	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5284	return VINF_SUCCESS;
5285
5286	VMMRZCallRing3Disable(pVCpu); /* We're not using hmR0VmxCallRing3Disable() as we want to respect nesting here. */
5287
5288	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5289	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5290
5291	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5292	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5293
5294	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5295	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5296
5297	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5298	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5299
5300	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5301	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5302
5303	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5304	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5305
5306	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5308
5309	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5311
5312	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
5313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestIntrState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5314
5315	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5317
5318	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5320
5321	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5322	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5323
5324	VMMRZCallRing3Enable(pVCpu);
5325	return rc;
5326	}
5327
5328
5329	/**
5330	* Check per-VM and per-VCPU force flag actions that require us to go back to
5331	* ring-3 for one reason or another.
5332	*
5333	* @returns VBox status code (information status code included).
5334	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5335	* ring-3.
5336	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5337	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5338	* interrupts)
5339	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5340	* all EMTs to be in ring-3.
5341	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5342	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5343	* to the EM loop.
5344	*
5345	* @param pVM Pointer to the VM.
5346	* @param pVCpu Pointer to the VMCPU.
5347	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5348	* out-of-sync. Make sure to update the required fields
5349	* before using them.
5350	*/
5351	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5352	{
5353	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5354
5355	int rc = VERR_INTERNAL_ERROR_5;
5356	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5357	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5358	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5359	{
5360	/* We need the control registers now, make sure the guest-CPU context is updated. */
5361	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5362	AssertRCReturn(rc, rc);
5363
5364	/* Pending HM CR3 sync. */
5365	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5366	{
5367	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5368	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5369	}
5370	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5371	{
5372	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5373	AssertRCReturn(rc, rc);
5374	}
5375
5376	/* Pending PGM C3 sync. */
5377	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5378	{
5379	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5380	if (rc != VINF_SUCCESS)
5381	{
5382	AssertRC(rc);
5383	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5384	return rc;
5385	}
5386	}
5387
5388	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5389	/* -XXX- what was that about single stepping? */
5390	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5391	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5392	{
5393	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5394	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5395	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5396	return rc;
5397	}
5398
5399	/* Pending VM request packets, such as hardware interrupts. */
5400	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5401	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5402	{
5403	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5404	return VINF_EM_PENDING_REQUEST;
5405	}
5406
5407	/* Pending PGM pool flushes. */
5408	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5409	{
5410	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5411	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5412	}
5413
5414	/* Pending DMA requests. */
5415	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5416	{
5417	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5418	return VINF_EM_RAW_TO_R3;
5419	}
5420	}
5421
5422	/* Paranoia. */
5423	Assert(rc != VERR_EM_INTERPRETER);
5424	return VINF_SUCCESS;
5425	}
5426
5427
5428	/**
5429	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5430	* VT-x to execute any instruction.
5431	*
5432	* @param pvCpu Pointer to the VMCPU.
5433	*/
5434	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5435	{
5436	if (pVCpu->hm.s.Event.fPending)
5437	{
5438	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5439	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5440	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5441	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5442
5443	/* If a trap was already pending, we did something wrong! */
5444	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5445
5446	/* A page-fault exception during a page-fault would become a double-fault. */
5447	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5448	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5449
5450	TRPMEVENT enmTrapType;
5451	switch (uVectorType)
5452	{
5453	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5454	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5455	enmTrapType = TRPM_HARDWARE_INT;
5456	break;
5457	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5458	enmTrapType = TRPM_SOFTWARE_INT;
5459	break;
5460	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5461	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5462	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5463	enmTrapType = TRPM_TRAP;
5464	break;
5465	default:
5466	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5467	enmTrapType = TRPM_32BIT_HACK;
5468	break;
5469	}
5470	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5471	AssertRC(rc);
5472	if (fErrorCodeValid)
5473	TRPMSetErrorCode(pVCpu, uErrorCode);
5474
5475	/* Clear the VT-x state bits now that TRPM has the information. */
5476	pVCpu->hm.s.Event.fPending = false;
5477	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5478	AssertRC(rc);
5479	}
5480	}
5481
5482
5483	/**
5484	* Does the necessary state syncing before doing a longjmp to ring-3.
5485	*
5486	* @param pVM Pointer to the VM.
5487	* @param pVCpu Pointer to the VMCPU.
5488	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5489	* out-of-sync. Make sure to update the required fields
5490	* before using them.
5491	* @param rcExit The reason for exiting to ring-3. Can be
5492	* VINF_VMM_UNKNOWN_RING3_CALL.
5493	*
5494	* @remarks No-long-jmp zone!!!
5495	*/
5496	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5497	{
5498	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5499	Log(("hmR0VmxLongJmpToRing3: rcExit=%d\n", rcExit));
5500
5501	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5502	AssertRC(rc);
5503
5504	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5505	if (CPUMIsGuestFPUStateActive(pVCpu))
5506	{
5507	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5508	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5509	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5510	}
5511
5512	/* Restore debug registers if necessary and resync on next R0 reentry. */
5513	if (CPUMIsGuestDebugStateActive(pVCpu))
5514	{
5515	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5516	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5517	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5518	}
5519	else if (CPUMIsHyperDebugStateActive(pVCpu))
5520	{
5521	CPUMR0LoadHostDebugState(pVM, pVCpu);
5522	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5523	}
5524
5525	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5526	}
5527
5528
5529	/**
5530	* An action requires us to go back to ring-3. This function does the necessary
5531	* steps before we can safely return to ring-3. This is not the same as longjmps
5532	* to ring-3, this is voluntary.
5533	*
5534	* @param pVM Pointer to the VM.
5535	* @param pVCpu Pointer to the VMCPU.
5536	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5537	* out-of-sync. Make sure to update the required fields
5538	* before using them.
5539	* @param rcExit The reason for exiting to ring-3. Can be
5540	* VINF_VMM_UNKNOWN_RING3_CALL.
5541	*/
5542	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5543	{
5544	Assert(pVM);
5545	Assert(pVCpu);
5546	Assert(pMixedCtx);
5547	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5548
5549	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5550	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5551	return;
5552
5553	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5554	VMMRZCallRing3Disable(pVCpu);
5555	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5556
5557	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5558	hmR0VmxUpdateTRPMTrap(pVCpu);
5559
5560	/* Sync. the guest state. */
5561	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5562	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5563
5564	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5565	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5566	\| CPUM_CHANGED_LDTR
5567	\| CPUM_CHANGED_GDTR
5568	\| CPUM_CHANGED_IDTR
5569	\| CPUM_CHANGED_TR
5570	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5571
5572	/* On our way back from ring-3 the following needs to be done. */
5573	/** @todo This can change with preemption hooks. */
5574	if (rcExit == VINF_EM_RAW_INTERRUPT)
5575	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5576	else
5577	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5578
5579	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5580	VMMRZCallRing3Enable(pVCpu);
5581	}
5582
5583
5584	/**
5585	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5586	* longjump to ring-3 and possibly get preempted.
5587	*
5588	* @param pVCpu Pointer to the VMCPU.
5589	* @param enmOperation The operation causing the ring-3 longjump.
5590	* @param pvUser The user argument (pointer to the possibly
5591	* out-of-date guest-CPU context).
5592	*
5593	* @remarks Must never be called with @a enmOperation ==
5594	* VMMCALLRING3_VM_R0_ASSERTION.
5595	*/
5596	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5597	{
5598	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5599	Assert(pVCpu);
5600	Assert(pvUser);
5601	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5602	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5603
5604	VMMRZCallRing3Disable(pVCpu);
5605	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5606	VMMRZCallRing3Enable(pVCpu);
5607	}
5608
5609
5610	/**
5611	* Injects any pending TRPM trap into the VM by updating the VMCS.
5612	*
5613	* @returns VBox status code (informational status code included).
5614	* @param pVM Pointer to the VM.
5615	* @param pVCpu Pointer to the VMCPU.
5616	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5617	* out-of-sync. Make sure to update the required fields
5618	* before using them.
5619	*/
5620	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5621	{
5622	if (!TRPMHasTrap(pVCpu))
5623	return VINF_SUCCESS;
5624
5625	uint8_t u8Vector = 0;
5626	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5627	RTGCUINT uErrCode = 0;
5628
5629	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5630	AssertRCReturn(rc, rc);
5631	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5632
5633	rc = TRPMResetTrap(pVCpu);
5634	AssertRCReturn(rc, rc);
5635
5636	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5637	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5638	if (enmTrpmEvent == TRPM_TRAP)
5639	{
5640	switch (u8Vector)
5641	{
5642	case X86_XCPT_BP:
5643	case X86_XCPT_OF:
5644	{
5645	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5646	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5647	break;
5648	}
5649
5650	case X86_XCPT_DF:
5651	case X86_XCPT_TS:
5652	case X86_XCPT_NP:
5653	case X86_XCPT_SS:
5654	case X86_XCPT_GP:
5655	case X86_XCPT_PF:
5656	case X86_XCPT_AC:
5657	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5658	them which VT-x/VMM pushes to the guest stack. */
5659	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5660	/* no break! */
5661	default:
5662	{
5663	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5664	break;
5665	}
5666	}
5667	}
5668	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5669	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5670	else
5671	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5672
5673	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5674	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5675	}
5676
5677
5678	/**
5679	* Checks if there are any pending guest interrupts to be delivered and injects
5680	* them into the VM by updating the VMCS.
5681	*
5682	* @returns VBox status code (informational status codes included).
5683	* @param pVM Pointer to the VM.
5684	* @param pVCpu Pointer to the VMCPU.
5685	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5686	* out-of-sync. Make sure to update the required fields
5687	* before using them.
5688	*
5689	* @remarks This must be called only after hmR0VmxSaveGuestIntrState().
5690	*/
5691	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5692	{
5693	/* First inject any pending HM interrupts. */
5694	if (pVCpu->hm.s.Event.fPending)
5695	{
5696	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5697	pVCpu->hm.s.Event.u32ErrCode);
5698	AssertRCReturn(rc, rc);
5699	pVCpu->hm.s.Event.fPending = false;
5700	return rc;
5701	}
5702
5703	/** @todo SMI. SMIs take priority over NMIs. */
5704
5705	/* NMI. NMIs take priority over regular interrupts . */
5706	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5707	{
5708	/* Construct an NMI interrupt and inject it into the VMCS. */
5709	RTGCUINTPTR uIntrInfo;
5710	uIntrInfo = X86_XCPT_NMI;
5711	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5712	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5713	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5714	AssertRCReturn(rc, rc);
5715	return rc;
5716	}
5717
5718	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5719	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5720	AssertRCReturn(rc, rc);
5721
5722	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5723	if (!TRPMHasTrap(pVCpu))
5724	{
5725	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5726	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5727	{
5728	/*
5729	* When external interrupts are pending and the guest has disabled interrupts, cause a VM-exit using "interrupt-window
5730	* exiting" so we can deliver the interrupt when the guest is ready to receive them. Otherwise, if the guest
5731	* can receive interrupts now, convert the PDM interrupt into a TRPM event and inject it.
5732	*/
5733	if (!(pMixedCtx->eflags.u32 & X86_EFL_IF)) /** @todo we can use interrupt-window exiting for block-by-STI. */
5734	{
5735	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5736	{
5737	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5738	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5739	AssertRCReturn(rc, rc);
5740	}
5741	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5742	}
5743	else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5744	{
5745	uint8_t u8Interrupt = 0;
5746	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5747	if (RT_SUCCESS(rc))
5748	{
5749	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5750	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5751	AssertRCReturn(rc, rc);
5752	}
5753	else
5754	{
5755	/** @todo Does this actually happen? If not turn it into an assertion. */
5756	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5757	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5758	}
5759	}
5760	}
5761	}
5762
5763	/* If interrupts can be delivered, inject it into the VM. */
5764	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5765	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5766	&& TRPMHasTrap(pVCpu))
5767	{
5768	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5769	AssertRCReturn(rc, rc);
5770	}
5771	return rc;
5772	}
5773
5774	/**
5775	* Injects an invalid-opcode (#UD) exception into the VM.
5776	*
5777	* @returns VBox status code (informational status code included).
5778	* @param pVM Pointer to the VM.
5779	* @param pVCpu Pointer to the VMCPU.
5780	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5781	* out-of-sync. Make sure to update the required fields
5782	* before using them.
5783	*/
5784	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5785	{
5786	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5787	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5788	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5789	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5790	}
5791
5792
5793	/**
5794	* Injects a double-fault (#DF) exception into the VM.
5795	*
5796	* @returns VBox status code (informational status code included).
5797	* @param pVM Pointer to the VM.
5798	* @param pVCpu Pointer to the VMCPU.
5799	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5800	* out-of-sync. Make sure to update the required fields
5801	* before using them.
5802	*/
5803	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5804	{
5805	/* Inject the double-fault. */
5806	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5807	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5808	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5809	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5810	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5811	}
5812
5813
5814	/**
5815	* Injects a debug (#DB) exception into the VM.
5816	*
5817	* @returns VBox status code (informational status code included).
5818	* @param pVM Pointer to the VM.
5819	* @param pVCpu Pointer to the VMCPU.
5820	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5821	* out-of-sync. Make sure to update the required fields
5822	* before using them.
5823	*/
5824	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5825	{
5826	/* Inject the debug-exception. */
5827	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5828	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5829	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5830	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5831	}
5832
5833
5834	/**
5835	* Injects a overflow (#OF) exception into the VM.
5836	*
5837	* @returns VBox status code (informational status code included).
5838	* @param pVM Pointer to the VM.
5839	* @param pVCpu Pointer to the VMCPU.
5840	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5841	* out-of-sync. Make sure to update the required fields
5842	* before using them.
5843	* @param cbInstr The value of RIP that is to be pushed on the guest
5844	* stack.
5845	*/
5846	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5847	{
5848	/* Inject the overflow exception. */
5849	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5850	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5851	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5852	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5853	}
5854
5855
5856	/**
5857	* Injects a general-protection (#GP) fault into the VM.
5858	*
5859	* @returns VBox status code (informational status code included).
5860	* @param pVM Pointer to the VM.
5861	* @param pVCpu Pointer to the VMCPU.
5862	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5863	* out-of-sync. Make sure to update the required fields
5864	* before using them.
5865	* @param u32ErrorCode The error code associated with the #GP.
5866	*/
5867	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5868	{
5869	/* Inject the general-protection fault. */
5870	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5871	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5872	if (fErrorCodeValid)
5873	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5874	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5875	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5876	}
5877
5878
5879	/**
5880	* Injects a software interrupt (INTn) into the VM.
5881	*
5882	* @returns VBox status code (informational status code included).
5883	* @param pVM Pointer to the VM.
5884	* @param pVCpu Pointer to the VMCPU.
5885	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5886	* out-of-sync. Make sure to update the required fields
5887	* before using them.
5888	* @param uVector The software interrupt vector number.
5889	* @param cbInstr The value of RIP that is to be pushed on the guest
5890	* stack.
5891	*/
5892	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5893	{
5894	/* Inject the INTn. */
5895	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5896	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5897	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5898	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5899	}
5900
5901
5902	/**
5903	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5904	* stack.
5905	*
5906	* @returns VBox status code (information status code included).
5907	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5908	* @param pVM Pointer to the VM.
5909	* @param pMixedCtx Pointer to the guest-CPU context.
5910	* @param uValue The value to push to the guest stack.
5911	*/
5912	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5913	{
5914	/*
5915	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5916	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5917	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5918	*/
5919	if (pMixedCtx->sp == 1)
5920	return VINF_EM_RESET;
5921	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5922	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5923	AssertRCReturn(rc, rc);
5924	return rc;
5925	}
5926
5927
5928	/**
5929	* Injects an event into the guest upon VM-entry by updating the relevant fields
5930	* in the VM-entry area in the VMCS.
5931	*
5932	* @returns VBox status code (informational error codes included).
5933	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5934	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5935	*
5936	* @param pVM Pointer to the VM.
5937	* @param pVCpu Pointer to the VMCPU.
5938	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5939	* out-of-sync. Make sure to update the required fields
5940	* before using them.
5941	* @param u64IntrInfo The VM-entry interruption-information field.
5942	* @param cbInstr The VM-entry instruction length in bytes (for software
5943	* interrupts, exceptions and privileged software
5944	* exceptions).
5945	* @param u32ErrCode The VM-entry exception error code.
5946	*
5947	* @remarks No-long-jump zone!!!
5948	*/
5949	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5950	uint32_t u32ErrCode)
5951	{
5952	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5953	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5954	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5955
5956	/* We require CR0 to check if the guest is in real-mode. */
5957	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5958	AssertRCReturn(rc, rc);
5959
5960	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5961	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5962
5963	/*
5964	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5965	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5966	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5967	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5968	*/
5969	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5970	{
5971	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5972	{
5973	Assert(PDMVmmDevHeapIsEnabled(pVM));
5974	Assert(pVM->hm.s.vmx.pRealModeTSS);
5975
5976	/* Save the required guest state bits from the VMCS. */
5977	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5978	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5979	AssertRCReturn(rc, rc);
5980
5981	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
5982	const size_t cbIdtEntry = 4;
5983	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
5984	{
5985	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
5986	if (uVector == X86_XCPT_DF)
5987	return VINF_EM_RESET;
5988	else if (uVector == X86_XCPT_GP)
5989	{
5990	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
5991	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
5992	}
5993
5994	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
5995	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
5996	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
5997	}
5998
5999	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6000	uint16_t uGuestIp = pMixedCtx->ip;
6001	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6002	{
6003	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6004	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6005	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6006	}
6007	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6008	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6009
6010	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6011	uint16_t offIdtEntry = 0;
6012	RTSEL selIdtEntry = 0;
6013	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6014	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6015	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6016	AssertRCReturn(rc, rc);
6017
6018	/* Construct the stack frame for the interrupt/exception handler. */
6019	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6020	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6021	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6022	AssertRCReturn(rc, rc);
6023
6024	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6025	if (rc == VINF_SUCCESS)
6026	{
6027	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6028	pMixedCtx->rip = offIdtEntry;
6029	pMixedCtx->cs.Sel = selIdtEntry;
6030	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6031
6032	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6033	\| HM_CHANGED_GUEST_RIP
6034	\| HM_CHANGED_GUEST_RFLAGS
6035	\| HM_CHANGED_GUEST_RSP;
6036	AssertRCReturn(rc, rc);
6037	}
6038	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6039	return rc;
6040	}
6041	else
6042	{
6043	/*
6044	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6045	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6046	*/
6047	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6048	}
6049	}
6050
6051	/* Add the valid bit, maybe the caller was lazy. */
6052	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6053
6054	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6055	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6056	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6057
6058	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6059	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6060	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6061	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6062	AssertRCReturn(rc, rc);
6063
6064	return rc;
6065	}
6066
6067
6068	/**
6069	* Enters the VT-x session.
6070	*
6071	* @returns VBox status code.
6072	* @param pVM Pointer to the VM.
6073	* @param pVCpu Pointer to the VMCPU.
6074	* @param pCpu Pointer to the CPU info struct.
6075	*/
6076	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6077	{
6078	AssertPtr(pVM);
6079	AssertPtr(pVCpu);
6080	Assert(pVM->hm.s.vmx.fSupported);
6081	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6082	NOREF(pCpu);
6083
6084	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6085
6086	/* Make sure we're in VMX root mode. */
6087	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6088	if (!(u32HostCR4 & X86_CR4_VMXE))
6089	{
6090	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6091	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6092	}
6093
6094	/* Load the active VMCS as the current one. */
6095	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6096	if (RT_FAILURE(rc))
6097	return rc;
6098
6099	/** @todo this will change with preemption hooks where can can VMRESUME as long
6100	* as we're no preempted. */
6101	pVCpu->hm.s.fResumeVM = false;
6102	return VINF_SUCCESS;
6103	}
6104
6105
6106	/**
6107	* Leaves the VT-x session.
6108	*
6109	* @returns VBox status code.
6110	* @param pVM Pointer to the VM.
6111	* @param pVCpu Pointer to the VMCPU.
6112	* @param pCtx Pointer to the guest-CPU context.
6113	*/
6114	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6115	{
6116	AssertPtr(pVCpu);
6117	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6118	NOREF(pVM);
6119	NOREF(pCtx);
6120
6121	/** @todo this will change with preemption hooks where we only VMCLEAR when
6122	* we are actually going to be preempted, not all the time like we
6123	* currently do. */
6124	/*
6125	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6126	* and mark the VMCS launch-state as "clear".
6127	*/
6128	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6129	return rc;
6130	}
6131
6132
6133	/**
6134	* Saves the host state in the VMCS host-state.
6135	* Sets up the VM-exit MSR-load area.
6136	*
6137	* The CPU state will be loaded from these fields on every successful VM-exit.
6138	*
6139	* @returns VBox status code.
6140	* @param pVM Pointer to the VM.
6141	* @param pVCpu Pointer to the VMCPU.
6142	*
6143	* @remarks No-long-jump zone!!!
6144	*/
6145	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6146	{
6147	AssertPtr(pVM);
6148	AssertPtr(pVCpu);
6149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6150
6151	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6152
6153	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6154	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6155	return VINF_SUCCESS;
6156
6157	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6158	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6159
6160	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6161	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6162
6163	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6164	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6165
6166	/* Reset the host-state-changed flag. */
6167	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6168	return rc;
6169	}
6170
6171
6172	/**
6173	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6174	* loaded from these fields on every successful VM-entry.
6175	*
6176	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6177	* Sets up the VM-entry controls.
6178	* Sets up the appropriate VMX non-root function to execute guest code based on
6179	* the guest CPU mode.
6180	*
6181	* @returns VBox status code.
6182	* @param pVM Pointer to the VM.
6183	* @param pVCpu Pointer to the VMCPU.
6184	* @param pCtx Pointer to the guest-CPU context.
6185	*
6186	* @remarks No-long-jump zone!!!
6187	*/
6188	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6189	{
6190	AssertPtr(pVM);
6191	AssertPtr(pVCpu);
6192	AssertPtr(pCtx);
6193	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6194
6195	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6196
6197	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6198
6199	/* Determine real-on-v86 mode. */
6200	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6201	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6202	&& CPUMIsGuestInRealModeEx(pCtx))
6203	{
6204	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6205	}
6206
6207	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6208	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6209
6210	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6211	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6212
6213	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6214	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6215
6216	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6217	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6218
6219	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6220	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6221
6222	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6223	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6224
6225	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6226	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6227
6228	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6229	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6230
6231	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6232	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6233
6234	rc = hmR0VmxLoadGuestIntrState(pVM, pVCpu, pCtx);
6235	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestIntrState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6236
6237	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6238	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6239
6240	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6241	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6242	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6243
6244	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6245	return rc;
6246	}
6247
6248
6249	/**
6250	* Does the preparations before executing guest code in VT-x.
6251	*
6252	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6253	* recompiler. We must be cautious what we do here regarding committing
6254	* guest-state information into the the VMCS assuming we assuredly execute the
6255	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6256	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6257	* that the recompiler can (and should) use them when it resumes guest
6258	* execution. Otherwise such operations must be done when we can no longer
6259	* exit to ring-3.
6260	*
6261	* @returns VBox status code (informational status codes included).
6262	* @retval VINF_SUCCESS if we can proceed with running the guest.
6263	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6264	* into the guest.
6265	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6266	*
6267	* @param pVM Pointer to the VM.
6268	* @param pVCpu Pointer to the VMCPU.
6269	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6270	* out-of-sync. Make sure to update the required fields
6271	* before using them.
6272	* @param pVmxTransient Pointer to the VMX transient structure.
6273	*
6274	* @remarks Called with preemption disabled.
6275	*/
6276	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6277	{
6278	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6279
6280	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6281	PGMRZDynMapFlushAutoSet(pVCpu);
6282	#endif
6283
6284	/* Check force flag actions that might require us to go back to ring-3. */
6285	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6286	if (rc != VINF_SUCCESS)
6287	return rc;
6288
6289	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6290	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6291	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6292	{
6293	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6294	RTGCPHYS GCPhysApicBase;
6295	GCPhysApicBase = pMixedCtx->msrApicBase;
6296	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6297
6298	/* Unalias any existing mapping. */
6299	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6300	AssertRCReturn(rc, rc);
6301
6302	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6303	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6304	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6305	AssertRCReturn(rc, rc);
6306
6307	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6308	}
6309
6310	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6311	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6312	pVmxTransient->uEFlags = ASMIntDisableFlags();
6313	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6314	{
6315	ASMSetFlags(pVmxTransient->uEFlags);
6316	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6317	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6318	return VINF_EM_RAW_INTERRUPT;
6319	}
6320	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6321	#endif
6322
6323	/*
6324	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6325	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6326	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6327	*/
6328	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6329	AssertRCReturn(rc, rc);
6330	return rc;
6331	}
6332
6333
6334	/**
6335	* Prepares to run guest code in VT-x and we've committed to doing so. This
6336	* means there is no backing out to ring-3 or anywhere else at this
6337	* point.
6338	*
6339	* @param pVM Pointer to the VM.
6340	* @param pVCpu Pointer to the VMCPU.
6341	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6342	* out-of-sync. Make sure to update the required fields
6343	* before using them.
6344	* @param pVmxTransient Pointer to the VMX transient structure.
6345	*
6346	* @remarks Called with preemption disabled.
6347	* @remarks No-long-jump zone!!!
6348	*/
6349	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6350	{
6351	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6352	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6353
6354	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6355	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6356	pVmxTransient->uEFlags = ASMIntDisableFlags();
6357	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6358	#endif
6359
6360	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6361	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6362	Log(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6363	int rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6364	AssertRC(rc);
6365	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6366
6367	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6368	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6369	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6370
6371	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6372	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6373	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6374
6375	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6376	if (pVmxTransient->fUpdateTscOffsetting)
6377	{
6378	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6379	pVmxTransient->fUpdateTscOffsetting = false;
6380	}
6381
6382	/*
6383	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6384	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6385	*/
6386	if (pVM->hm.s.fTPRPatchingActive)
6387	{
6388	Assert(!CPUMIsGuestInLongMode(pVCpu));
6389
6390	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6391	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6392	AssertRC(rc);
6393
6394	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6395	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6396	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6397	see hmR0VmxLoadGuestApicState(). */
6398	}
6399
6400	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6401	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6402	to start executing. */
6403	}
6404
6405
6406	/**
6407	* Performs some essential restoration of state after running guest code in
6408	* VT-x.
6409	*
6410	* @param pVM Pointer to the VM.
6411	* @param pVCpu Pointer to the VMCPU.
6412	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6413	* out-of-sync. Make sure to update the required fields
6414	* before using them.
6415	* @param pVmxTransient Pointer to the VMX transient structure.
6416	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6417	*
6418	* @remarks Called with interrupts disabled.
6419	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6420	* unconditionally when it is safe to do so.
6421	*/
6422	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6423	{
6424	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6425	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6426
6427	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6428	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6429	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6430	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6431	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6432
6433	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6434	{
6435	/** @todo Find a way to fix hardcoding a guestimate. */
6436	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6437	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6438	}
6439
6440	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6441	Assert(!(ASMGetFlags() & X86_EFL_IF));
6442	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6443
6444	/* Restore the effects of TPR patching if any. */
6445	if (pVM->hm.s.fTPRPatchingActive)
6446	{
6447	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6448	AssertRC(rc);
6449	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6450	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6451	}
6452
6453	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6454	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6455
6456	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6457	uint32_t uExitReason;
6458	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6459	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6460	AssertRC(rc);
6461	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6462	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6463
6464	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6465	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6466
6467	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6468	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6469	{
6470	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6471	return;
6472	}
6473
6474	/* We need to update our interruptibility-state on every VM-exit and VM-entry. */
6475	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6476	AssertRC(rc);
6477	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE;
6478
6479	/*
6480	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever we
6481	* eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why we do
6482	* it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6483	*/
6484	if ( !pVmxTransient->fVMEntryFailed
6485	&& (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6486	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6487	{
6488	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6489	AssertRC(rc);
6490	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6491	}
6492	}
6493
6494
6495	/**
6496	* Runs the guest code using VT-x.
6497	*
6498	* @returns VBox status code.
6499	* @param pVM Pointer to the VM.
6500	* @param pVCpu Pointer to the VMCPU.
6501	* @param pCtx Pointer to the guest-CPU context.
6502	*
6503	* @remarks Called with preemption disabled.
6504	*/
6505	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6506	{
6507	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6508	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6509
6510	VMXTRANSIENT VmxTransient;
6511	VmxTransient.fUpdateTscOffsetting = true;
6512	int rc = VERR_INTERNAL_ERROR_5;
6513	unsigned cLoops = 0;
6514
6515	for (;; cLoops++)
6516	{
6517	Assert(!HMR0SuspendPending());
6518	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6519	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6520	(unsigned)RTMpCpuId(), cLoops));
6521
6522	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6523	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6524	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6525	if (rc != VINF_SUCCESS)
6526	break;
6527
6528	/*
6529	* No longjmps to ring-3 from this point on!!!
6530	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6531	* This also disables flushing of the R0-logger instance (if any).
6532	*/
6533	VMMRZCallRing3Disable(pVCpu);
6534	VMMRZCallRing3RemoveNotification(pVCpu);
6535	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6536
6537	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6538	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6539
6540	/*
6541	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6542	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6543	*/
6544	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6545	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6546	{
6547	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6548	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6549	return rc;
6550	}
6551
6552	/* Handle the VM-exit. */
6553	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6554	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6555	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6556	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6557	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6558	if (rc != VINF_SUCCESS)
6559	break;
6560	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6561	{
6562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6563	rc = VINF_EM_RAW_INTERRUPT;
6564	break;
6565	}
6566	}
6567
6568	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6569	if (rc == VERR_EM_INTERPRETER)
6570	rc = VINF_EM_RAW_EMULATE_INSTR;
6571	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6572	return rc;
6573	}
6574
6575	#ifdef DEBUG
6576	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6577	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6578	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6579	# define VMX_ASSERT_PREEMPT_CPUID() \
6580	do \
6581	{ \
6582	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6583	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6584	} while (0)
6585
6586	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6587	do { \
6588	AssertPtr(pVM); \
6589	AssertPtr(pVCpu); \
6590	AssertPtr(pMixedCtx); \
6591	AssertPtr(pVmxTransient); \
6592	Assert(pVmxTransient->fVMEntryFailed == false); \
6593	Assert(ASMIntAreEnabled() == true); \
6594	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6595	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6596	LogFunc(("\n")); \
6597	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6598	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6599	VMX_ASSERT_PREEMPT_CPUID(); \
6600	} while (0)
6601	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6602	do { \
6603	LogFunc(("\n")); \
6604	} while(0)
6605	#else /* Release builds */
6606	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6607	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6608	#endif
6609
6610
6611	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6612	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6613	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6614	/**
6615	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6616	*/
6617	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6618	{
6619	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6621	return VINF_SUCCESS;
6622	}
6623
6624
6625	/**
6626	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6627	*/
6628	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6629	{
6630	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6631	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6632	AssertRCReturn(rc, rc);
6633
6634	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6635	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6636	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6637
6638	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6639	return VINF_EM_RAW_INTERRUPT;
6640
6641	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6642	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
6643	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6644	return VINF_SUCCESS;
6645	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6646	return rc;
6647
6648	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6649	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6650	switch (uIntrType)
6651	{
6652	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6653	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6654	/* no break */
6655	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6656	{
6657	switch (uVector)
6658	{
6659	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6660	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6661	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6662	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6663	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6664	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6665	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6666	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6667	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6668	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6669	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6670	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6671	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6672	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6673	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6674	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6675	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6676	#endif
6677	default:
6678	{
6679	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6680	AssertRCReturn(rc, rc);
6681
6682	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6683	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6684	{
6685	Assert(pVM->hm.s.vmx.pRealModeTSS);
6686	Assert(PDMVmmDevHeapIsEnabled(pVM));
6687	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6688	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6689	AssertRCReturn(rc, rc);
6690	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6691	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6692	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6693	AssertRCReturn(rc, rc);
6694	}
6695	else
6696	{
6697	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6698	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6699	}
6700	break;
6701	}
6702	}
6703	break;
6704	}
6705
6706	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6707	default:
6708	{
6709	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6710	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6711	break;
6712	}
6713	}
6714	return rc;
6715	}
6716
6717
6718	/**
6719	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6720	*/
6721	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6722	{
6723	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6724	int rc = VERR_INTERNAL_ERROR_5;
6725	#ifdef DEBUG
6726	rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6727	AssertRC(rc);
6728	Assert(pMixedCtx->eflags.u32 & X86_EFL_IF);
6729	#endif
6730
6731	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6732	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6733	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6734	AssertRCReturn(rc, rc);
6735
6736	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6737	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6738	return VINF_SUCCESS;
6739	}
6740
6741
6742	/**
6743	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6744	*/
6745	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6746	{
6747	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6748	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6749	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6750	}
6751
6752
6753	/**
6754	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6755	*/
6756	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6757	{
6758	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6759	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6760	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6761	AssertRCReturn(rc, rc);
6762
6763	pMixedCtx->rip += pVmxTransient->cbInstr;
6764	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6765
6766	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6767	return VINF_SUCCESS;
6768	}
6769
6770
6771	/**
6772	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6773	*/
6774	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6775	{
6776	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6777	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6778	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6779	AssertRCReturn(rc, rc);
6780
6781	pMixedCtx->rip += pVmxTransient->cbInstr;
6782	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6783
6784	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6785	return VINF_SUCCESS;
6786	}
6787
6788
6789	/**
6790	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6791	*/
6792	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6793	{
6794	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6795	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6796	if (RT_LIKELY(rc == VINF_SUCCESS))
6797	{
6798	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6799	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6800	AssertRCReturn(rc, rc);
6801	Assert(pVmxTransient->cbInstr == 2);
6802
6803	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6804	pMixedCtx->rip += pVmxTransient->cbInstr;
6805	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6806	}
6807	else
6808	{
6809	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6810	rc = VERR_EM_INTERPRETER;
6811	}
6812	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6813	return rc;
6814	}
6815
6816
6817	/**
6818	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6819	*/
6820	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6821	{
6822	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6823	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6824	AssertRCReturn(rc, rc);
6825
6826	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6827	return VINF_EM_RAW_EMULATE_INSTR;
6828
6829	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6830	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6831	}
6832
6833
6834	/**
6835	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6836	*/
6837	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6838	{
6839	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6840	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6841	AssertRCReturn(rc, rc);
6842
6843	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6844	if (RT_LIKELY(rc == VINF_SUCCESS))
6845	{
6846	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6847	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6848	AssertRCReturn(rc, rc);
6849	Assert(pVmxTransient->cbInstr == 2);
6850
6851	pMixedCtx->rip += pVmxTransient->cbInstr;
6852	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6853
6854	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6855	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6856	pVmxTransient->fUpdateTscOffsetting = true;
6857	}
6858	else
6859	{
6860	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6861	rc = VERR_EM_INTERPRETER;
6862	}
6863	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6864	return rc;
6865	}
6866
6867
6868	/**
6869	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6870	*/
6871	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6872	{
6873	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6874	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6875	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6876	AssertRCReturn(rc, rc);
6877
6878	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6879	if (RT_LIKELY(rc == VINF_SUCCESS))
6880	{
6881	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6882	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6883	AssertRCReturn(rc, rc);
6884	Assert(pVmxTransient->cbInstr == 3);
6885
6886	pMixedCtx->rip += pVmxTransient->cbInstr;
6887	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6888
6889	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6890	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6891	pVmxTransient->fUpdateTscOffsetting = true;
6892	}
6893	else
6894	{
6895	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6896	rc = VERR_EM_INTERPRETER;
6897	}
6898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6899	return rc;
6900	}
6901
6902
6903	/**
6904	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6905	*/
6906	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6907	{
6908	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6909	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6910	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6911	AssertRCReturn(rc, rc);
6912
6913	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6914	if (RT_LIKELY(rc == VINF_SUCCESS))
6915	{
6916	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6917	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6918	AssertRCReturn(rc, rc);
6919	Assert(pVmxTransient->cbInstr == 2);
6920
6921	pMixedCtx->rip += pVmxTransient->cbInstr;
6922	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6923	}
6924	else
6925	{
6926	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6927	rc = VERR_EM_INTERPRETER;
6928	}
6929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6930	return rc;
6931	}
6932
6933
6934	/**
6935	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6936	*/
6937	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6938	{
6939	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6940	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6941	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6942	AssertRCReturn(rc, rc);
6943
6944	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6945	rc = VBOXSTRICTRC_VAL(rc2);
6946	if (RT_LIKELY(rc == VINF_SUCCESS))
6947	{
6948	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6949	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6950	AssertRCReturn(rc, rc);
6951
6952	pMixedCtx->rip += pVmxTransient->cbInstr;
6953	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6954	}
6955	else
6956	{
6957	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6958	pVmxTransient->uExitQualification, rc));
6959	rc = VERR_EM_INTERPRETER;
6960	}
6961	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6962	return rc;
6963	}
6964
6965
6966	/**
6967	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6968	*/
6969	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6970	{
6971	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6972	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6973	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6974	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6975	AssertRCReturn(rc, rc);
6976
6977	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6978	if (RT_LIKELY(rc == VINF_SUCCESS))
6979	{
6980	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6981	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6982	AssertRCReturn(rc, rc);
6983
6984	pMixedCtx->rip += pVmxTransient->cbInstr;
6985	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6986	}
6987	else
6988	{
6989	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
6990	rc = VERR_EM_INTERPRETER;
6991	}
6992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
6993	return rc;
6994	}
6995
6996
6997	/**
6998	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
6999	*/
7000	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7001	{
7002	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7003	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7004	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7005	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7006	AssertRCReturn(rc, rc);
7007
7008	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7009	rc = VBOXSTRICTRC_VAL(rc2);
7010	if (RT_LIKELY( rc == VINF_SUCCESS
7011	\|\| rc == VINF_EM_HALT))
7012	{
7013	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7014	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7015	AssertRCReturn(rc3, rc3);
7016
7017	pMixedCtx->rip += pVmxTransient->cbInstr;
7018	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7019
7020	if ( rc == VINF_EM_HALT
7021	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7022	{
7023	rc = VINF_SUCCESS;
7024	}
7025	}
7026	else
7027	{
7028	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7029	rc = VERR_EM_INTERPRETER;
7030	}
7031	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7032	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7034	return rc;
7035	}
7036
7037
7038	/**
7039	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7040	*/
7041	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7042	{
7043	/*
7044	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7045	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7046	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7047	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7048	*/
7049	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7050	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7051	}
7052
7053
7054	/**
7055	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7056	*/
7057	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7058	{
7059	/*
7060	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7061	* root operation. If we get there there is something funny going on.
7062	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7063	*/
7064	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7065	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7066	}
7067
7068
7069	/**
7070	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7071	*/
7072	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7073	{
7074	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7075	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7076	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7077	}
7078
7079
7080	/**
7081	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7082	*/
7083	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7084	{
7085	/*
7086	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7087	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7088	* See Intel spec. 25.3 "Other Causes of VM-exits".
7089	*/
7090	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7091	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7092	}
7093
7094
7095	/**
7096	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7097	* VM-exit.
7098	*/
7099	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7100	{
7101	/*
7102	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7103	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7104	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7105	*/
7106	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7107	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7108	}
7109
7110
7111	/**
7112	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7113	* VM-exit.
7114	*/
7115	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7116	{
7117	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7118	return VINF_EM_RESET;
7119	}
7120
7121
7122	/**
7123	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7124	*/
7125	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7126	{
7127	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7128	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7129	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7130	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7131	AssertRCReturn(rc, rc);
7132
7133	pMixedCtx->rip++;
7134	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7135	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7136	rc = VINF_SUCCESS;
7137	else
7138	rc = VINF_EM_HALT;
7139
7140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7141	return rc;
7142	}
7143
7144
7145	/**
7146	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7147	*/
7148	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7149	{
7150	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7151	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7152	AssertRCReturn(rc, rc);
7153	return rc;
7154	}
7155
7156
7157	/**
7158	* VM-exit handler for expiry of the VMX preemption timer.
7159	*/
7160	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7161	{
7162	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7163	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7164	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7165	int rc = fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7166	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7167	return rc;
7168	}
7169
7170
7171	/**
7172	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7173	*/
7174	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7175	{
7176	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7177	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7178	/** @todo check if XSETBV is supported by the recompiler. */
7179	return VERR_EM_INTERPRETER;
7180	}
7181
7182
7183	/**
7184	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7185	*/
7186	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7187	{
7188	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7189	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7190	/** @todo implement EMInterpretInvpcid() */
7191	return VERR_EM_INTERPRETER;
7192	}
7193
7194
7195	/**
7196	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7197	* Error VM-exit.
7198	*/
7199	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7200	{
7201	uint32_t uIntrState;
7202	RTHCUINTREG uHCReg;
7203	uint64_t u64Val;
7204
7205	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7206	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7207	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7208	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7209	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7210	AssertRCReturn(rc, rc);
7211
7212	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7213	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7214	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7215	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7216
7217	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7218	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7219	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7220	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7221	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7222	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7223	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7224	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7225	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7226	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7227	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7228	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7229
7230	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7231
7232	return VERR_VMX_INVALID_GUEST_STATE;
7233	}
7234
7235
7236	/**
7237	* VM-exit handler for VM-entry failure due to an MSR-load
7238	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7239	*/
7240	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7241	{
7242	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7243	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7244	}
7245
7246
7247	/**
7248	* VM-exit handler for VM-entry failure due to a machine-check event
7249	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7250	*/
7251	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7252	{
7253	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7254	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7255	}
7256
7257
7258	/**
7259	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7260	* theory.
7261	*/
7262	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7263	{
7264	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7265	pVM, pVCpu, pMixedCtx));
7266	return VERR_VMX_UNDEFINED_EXIT_CODE;
7267	}
7268
7269
7270	/**
7271	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7272	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7273	* Conditional VM-exit.
7274	*/
7275	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7276	{
7277	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7278	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7279	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7280	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7281	return VERR_EM_INTERPRETER;
7282	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7283	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7284	}
7285
7286
7287	/**
7288	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7289	*/
7290	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7291	{
7292	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7293	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7294	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7295	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7296	return VERR_EM_INTERPRETER;
7297	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7298	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7299	}
7300
7301
7302	/**
7303	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7304	*/
7305	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7306	{
7307	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7308	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7309	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7310	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7311	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7312	AssertRCReturn(rc, rc);
7313
7314	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7315	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7316	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7318
7319	/* Update RIP and continue guest execution. */
7320	if (RT_LIKELY(rc == VINF_SUCCESS))
7321	{
7322	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7323	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7324	AssertRCReturn(rc, rc);
7325
7326	Assert(pVmxTransient->cbInstr == 2);
7327	pMixedCtx->rip += pVmxTransient->cbInstr;
7328	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7329	}
7330	return rc;
7331	}
7332
7333
7334	/**
7335	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7336	*/
7337	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7338	{
7339	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7340	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7341	AssertRCReturn(rc, rc);
7342	Assert(pVmxTransient->cbInstr == 2);
7343
7344	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7345	if ( pVM->hm.s.fTPRPatchingActive
7346	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7347	{
7348	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7349	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7350	{
7351	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7352	AssertRC(rc);
7353	}
7354
7355	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7356	AssertRCReturn(rc, rc);
7357	pMixedCtx->rip += pVmxTransient->cbInstr;
7358	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7360	return VINF_SUCCESS;
7361	}
7362
7363	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7364	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7365	{
7366	switch (pMixedCtx->ecx)
7367	{
7368	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7369	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7370	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7371	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7372	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7373	}
7374	}
7375	#ifdef DEBUG
7376	else
7377	{
7378	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7379	switch (pMixedCtx->ecx)
7380	{
7381	case MSR_IA32_SYSENTER_CS:
7382	case MSR_IA32_SYSENTER_EIP:
7383	case MSR_IA32_SYSENTER_ESP:
7384	case MSR_K8_FS_BASE:
7385	case MSR_K8_GS_BASE:
7386	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7387	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7388	case MSR_K8_LSTAR:
7389	case MSR_K6_STAR:
7390	case MSR_K8_SF_MASK:
7391	case MSR_K8_TSC_AUX:
7392	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7393	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7394	}
7395	}
7396	#endif
7397
7398	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7399	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7400	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7401	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7402	AssertRCReturn(rc, rc);
7403
7404	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7405	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7406	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7407
7408	/* Update guest-state and continue execution. */
7409	if (RT_LIKELY(rc == VINF_SUCCESS))
7410	{
7411	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7412	AssertRCReturn(rc, rc);
7413
7414	pMixedCtx->rip += pVmxTransient->cbInstr;
7415	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7416
7417	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7418	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7419	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7420	{
7421	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7422	}
7423	}
7424	return rc;
7425	}
7426
7427
7428	/**
7429	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7430	*/
7431	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7432	{
7433	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7434	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7435	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7436	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7437	return VERR_EM_INTERPRETER;
7438	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7439	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7440	}
7441
7442
7443	/**
7444	* VM-exit handler for when the TPR value is lowered below the specified
7445	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7446	*/
7447	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7448	{
7449	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7450	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7451
7452	/*
7453	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7454	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7455	* resume guest execution.
7456	*/
7457	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7459	return VINF_SUCCESS;
7460	}
7461
7462
7463	/**
7464	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7465	* VM-exit.
7466	*
7467	* @retval VINF_SUCCESS when guest execution can continue.
7468	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7469	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7470	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7471	* recompiler.
7472	*/
7473	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7474	{
7475	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7476	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7477	AssertRCReturn(rc, rc);
7478
7479	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7480	const uint8_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7481	switch (uAccessType)
7482	{
7483	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7484	{
7485	#if 0
7486	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7487	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7488	#else
7489	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7490	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7491	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7492	#endif
7493	AssertRCReturn(rc, rc);
7494
7495	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7496	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7497	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7498	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7499
7500	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7501	{
7502	case 0: /* CR0 */
7503	Log(("CR0 write rc=%d\n", rc));
7504	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7505	break;
7506	case 2: /* CR2 */
7507	Log(("CR2 write rc=%d\n", rc));
7508	break;
7509	case 3: /* CR3 */
7510	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7511	Log(("CR3 write rc=%d\n", rc));
7512	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7513	break;
7514	case 4: /* CR4 */
7515	Log(("CR4 write rc=%d\n", rc));
7516	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7517	break;
7518	case 8: /* CR8 */
7519	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7520	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7521	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7522	break;
7523	default:
7524	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7525	break;
7526	}
7527
7528	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7529	break;
7530	}
7531
7532	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7533	{
7534	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7535	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7536	AssertRCReturn(rc, rc);
7537	Assert( !pVM->hm.s.fNestedPaging
7538	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7539	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7540
7541	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7542	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7543	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7544
7545	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7546	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7547	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7548	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7549	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7550	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7551	break;
7552	}
7553
7554	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7555	{
7556	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7557	rc \|= EMInterpretCLTS(pVM, pVCpu);
7558	AssertRCReturn(rc, rc);
7559	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7560	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7561	Log(("CRX CLTS write rc=%d\n", rc));
7562	break;
7563	}
7564
7565	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7566	{
7567	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7568	AssertRCReturn(rc, rc);
7569	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7570	if (RT_LIKELY(rc == VINF_SUCCESS))
7571	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7573	Log(("CRX LMSW write rc=%d\n", rc));
7574	break;
7575	}
7576
7577	default:
7578	{
7579	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7580	return VERR_VMX_UNEXPECTED_EXCEPTION;
7581	}
7582	}
7583
7584	/* Validate possible error codes. */
7585	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7586	if (RT_SUCCESS(rc))
7587	{
7588	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7589	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7590	AssertRCReturn(rc2, rc2);
7591	pMixedCtx->rip += pVmxTransient->cbInstr;
7592	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7593	}
7594
7595	return rc;
7596	}
7597
7598
7599	/**
7600	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7601	* VM-exit.
7602	*/
7603	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7604	{
7605	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7606
7607	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7608	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7609	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7610	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7611	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7612	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7613	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7614	AssertRCReturn(rc, rc);
7615
7616	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7617	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7618	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7619	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7620	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7621	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7622	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7623
7624	/* I/O operation lookup arrays. */
7625	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7626	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7627
7628	const uint32_t cbSize = s_aIOSize[uIOWidth];
7629	const uint32_t cbInstr = pVmxTransient->cbInstr;
7630	if (fIOString)
7631	{
7632	/* INS/OUTS - I/O String instruction. */
7633	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7634	/** @todo for now manually disassemble later optimize by getting the fields from
7635	* the VMCS. */
7636	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7637	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7638	* segment prefix info. */
7639	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7640	if (RT_SUCCESS(rc))
7641	{
7642	if (fIOWrite)
7643	{
7644	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7645	(DISCPUMODE)pDis->uAddrMode, cbSize);
7646	rc = VBOXSTRICTRC_VAL(rc2);
7647	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7648	}
7649	else
7650	{
7651	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7652	(DISCPUMODE)pDis->uAddrMode, cbSize);
7653	rc = VBOXSTRICTRC_VAL(rc2);
7654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7655	}
7656	}
7657	else
7658	{
7659	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7660	rc = VINF_EM_RAW_EMULATE_INSTR;
7661	}
7662	}
7663	else
7664	{
7665	/* IN/OUT - I/O instruction. */
7666	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7667	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7668	if (fIOWrite)
7669	{
7670	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7671	rc = VBOXSTRICTRC_VAL(rc2);
7672	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7673	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7675	}
7676	else
7677	{
7678	uint32_t u32Result = 0;
7679	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7680	rc = VBOXSTRICTRC_VAL(rc2);
7681	if (IOM_SUCCESS(rc))
7682	{
7683	/* Save result of I/O IN instr. in AL/AX/EAX. */
7684	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7685	}
7686	else if (rc == VINF_IOM_R3_IOPORT_READ)
7687	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7689	}
7690	}
7691
7692	if (IOM_SUCCESS(rc))
7693	{
7694	pMixedCtx->rip += cbInstr;
7695	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7696	if (RT_LIKELY(rc == VINF_SUCCESS))
7697	{
7698	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7699	AssertRCReturn(rc, rc);
7700
7701	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7702	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7703	{
7704	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7705	for (unsigned i = 0; i < 4; i++)
7706	{
7707	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7708	if ( ( uIOPort >= pMixedCtx->dr[i]
7709	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7710	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7711	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7712	{
7713	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7714	uint64_t uDR6 = ASMGetDR6();
7715
7716	/* Clear all breakpoint status flags and set the one we just hit. */
7717	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7718	uDR6 \|= (uint64_t)RT_BIT(i);
7719
7720	/*
7721	* Note: AMD64 Architecture Programmer's Manual 13.1:
7722	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7723	* be cleared by software after the contents have been read.
7724	*/
7725	ASMSetDR6(uDR6);
7726
7727	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7728	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7729
7730	/* Paranoia. */
7731	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7732	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7733	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7734
7735	/* Resync DR7 */
7736	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7737	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7738
7739	/* Inject #DB and get on with guest execution. */
7740	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7741	AssertRCReturn(rc, rc);
7742	break;
7743	}
7744	}
7745	}
7746	}
7747	}
7748
7749	#ifdef DEBUG
7750	if (rc == VINF_IOM_R3_IOPORT_READ)
7751	Assert(!fIOWrite);
7752	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7753	Assert(fIOWrite);
7754	else
7755	{
7756	AssertMsg( RT_FAILURE(rc)
7757	\|\| rc == VINF_SUCCESS
7758	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7759	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7760	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7761	}
7762	#endif
7763
7764	return rc;
7765	}
7766
7767
7768	/**
7769	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7770	* VM-exit.
7771	*/
7772	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7773	{
7774	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7775
7776	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
7777	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7778	AssertRCReturn(rc, rc);
7779	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
7780	{
7781	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
7782	AssertRCReturn(rc, rc);
7783	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
7784	{
7785	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
7786	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7787	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7788	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7789	{
7790	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
7791	pVCpu->hm.s.Event.fPending = true;
7792	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
7793	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
7794	AssertRCReturn(rc, rc);
7795	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
7796	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
7797	else
7798	pVCpu->hm.s.Event.u32ErrCode = 0;
7799	}
7800	}
7801	}
7802	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7803	* emulation. */
7804	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7805	return VERR_EM_INTERPRETER;
7806	}
7807
7808
7809	/**
7810	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7811	*/
7812	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7813	{
7814	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7815	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7816	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7817	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7818	AssertRCReturn(rc, rc);
7819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7820	return VINF_EM_DBG_STOP;
7821	}
7822
7823
7824	/**
7825	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7826	*/
7827	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7828	{
7829	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7830	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7831
7832	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7833	rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7834	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7835	return VINF_SUCCESS;
7836	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7837	return rc;
7838
7839	#if 0
7840	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7841	* just sync the whole thing. */
7842	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7843	#else
7844	/* Aggressive state sync. for now. */
7845	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
7846	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
7847	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7848	#endif
7849	AssertRCReturn(rc, rc);
7850
7851	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7852	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7853	switch (uAccessType)
7854	{
7855	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7856	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7857	{
7858	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7859	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7860	{
7861	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7862	}
7863
7864	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7865	GCPhys &= PAGE_BASE_GC_MASK;
7866	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7867	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7868	CPUMCTX2CORE(pMixedCtx), GCPhys);
7869	rc = VBOXSTRICTRC_VAL(rc2);
7870	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7871	if ( rc == VINF_SUCCESS
7872	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7873	\|\| rc == VERR_PAGE_NOT_PRESENT)
7874	{
7875	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7876	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7877	rc = VINF_SUCCESS;
7878	}
7879	break;
7880	}
7881
7882	default:
7883	rc = VINF_EM_RAW_EMULATE_INSTR;
7884	break;
7885	}
7886
7887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7888	return rc;
7889	}
7890
7891
7892	/**
7893	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7894	* VM-exit.
7895	*/
7896	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7897	{
7898	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7899
7900	/* We should -not- get this VM-exit if the guest is debugging. */
7901	if (CPUMIsGuestDebugStateActive(pVCpu))
7902	{
7903	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7904	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7905	}
7906
7907	int rc = VERR_INTERNAL_ERROR_5;
7908	if ( !DBGFIsStepping(pVCpu)
7909	&& !CPUMIsHyperDebugStateActive(pVCpu))
7910	{
7911	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7912
7913	/* Don't intercept MOV DRx. */
7914	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7915	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7916	AssertRCReturn(rc, rc);
7917
7918	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7919	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7920	AssertRCReturn(rc, rc);
7921	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7922
7923	#ifdef VBOX_WITH_STATISTICS
7924	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7925	AssertRCReturn(rc, rc);
7926	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7928	else
7929	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7930	#endif
7931	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7932	return VINF_SUCCESS;
7933	}
7934
7935	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7936	* time and restore DRx registers afterwards */
7937	/*
7938	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7939	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7940	*/
7941	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7942	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7943	AssertRCReturn(rc, rc);
7944
7945	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7946	{
7947	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7948	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7949	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7950	if (RT_SUCCESS(rc))
7951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7953	}
7954	else
7955	{
7956	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7957	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7958	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7959	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7960	}
7961
7962	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7963	if (RT_SUCCESS(rc))
7964	{
7965	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7966	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7967	AssertRCReturn(rc2, rc2);
7968	pMixedCtx->rip += pVmxTransient->cbInstr;
7969	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7970	}
7971	return rc;
7972	}
7973
7974
7975	/**
7976	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7977	* Conditional VM-exit.
7978	*/
7979	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7980	{
7981	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7982	Assert(pVM->hm.s.fNestedPaging);
7983
7984	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7985	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
7986	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7987	return VINF_SUCCESS;
7988	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7989	return rc;
7990
7991	RTGCPHYS GCPhys = 0;
7992	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
7993	AssertRCReturn(rc, rc);
7994
7995	#if 0
7996	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
7997	#else
7998	/* Aggressive state sync. for now. */
7999	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8000	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8001	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8002	#endif
8003	AssertRCReturn(rc, rc);
8004
8005	/*
8006	* If we succeed, resume guest execution.
8007	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8008	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8009	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8010	* weird case. See @bugref{6043}.
8011	*/
8012	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8013	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8014	rc = VBOXSTRICTRC_VAL(rc2);
8015	if ( rc == VINF_SUCCESS
8016	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8017	\|\| rc == VERR_PAGE_NOT_PRESENT)
8018	{
8019	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8020	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8021	return VINF_SUCCESS;
8022	}
8023	return rc;
8024	}
8025
8026
8027	/**
8028	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8029	* VM-exit.
8030	*/
8031	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8032	{
8033	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8034	Assert(pVM->hm.s.fNestedPaging);
8035
8036	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8037	int rc = hmR0VmxCheckExitDueToEventDelivery(pVM, pVCpu, pMixedCtx, pVmxTransient);
8038	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8039	return VINF_SUCCESS;
8040	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8041	return rc;
8042
8043	RTGCPHYS GCPhys = 0;
8044	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8045	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8046	#if 0
8047	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
8048	#else
8049	/* Aggressive state sync. for now. */
8050	rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
8051	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8052	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8053	#endif
8054	AssertRCReturn(rc, rc);
8055
8056	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8057	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8058
8059	RTGCUINT uErrorCode = 0;
8060	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8061	uErrorCode \|= X86_TRAP_PF_ID;
8062	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8063	uErrorCode \|= X86_TRAP_PF_RW;
8064	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8065	uErrorCode \|= X86_TRAP_PF_P;
8066
8067	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8068	TRPMSetErrorCode(pVCpu, uErrorCode);
8069	TRPMSetFaultAddress(pVCpu, GCPhys);
8070
8071	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8072	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8073
8074	/* Handle the pagefault trap for the nested shadow table. */
8075	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8076	TRPMResetTrap(pVCpu);
8077
8078	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8079	if ( rc == VINF_SUCCESS
8080	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8081	\|\| rc == VERR_PAGE_NOT_PRESENT)
8082	{
8083	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8085	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8086	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8087	return VINF_SUCCESS;
8088	}
8089
8090	Log(("EPT return to ring-3 rc=%d\n"));
8091
8092	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8093	/** @todo Shouldn't we update TRPM here? */
8094	return rc;
8095	}
8096
8097
8098	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8099	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8100	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8101	/**
8102	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8103	*/
8104	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8105	{
8106	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8107
8108	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8109	AssertRCReturn(rc, rc);
8110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8111
8112	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8113	{
8114	/* Old-style FPU error reporting needs some extra work. */
8115	/** @todo don't fall back to the recompiler, but do it manually. */
8116	return VERR_EM_INTERPRETER;
8117	}
8118	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8119	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8120	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8121	AssertRCReturn(rc, rc);
8122	return rc;
8123	}
8124
8125
8126	/**
8127	* VM-exit exception handler for #BP (Breakpoint exception).
8128	*/
8129	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8130	{
8131	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8132
8133	/** @todo Try optimize this by not saving the entire guest state unless
8134	* really needed. */
8135	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8136	AssertRCReturn(rc, rc);
8137	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8138
8139	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8140	if (rc == VINF_EM_RAW_GUEST_TRAP)
8141	{
8142	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8143	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8144	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8145	AssertRCReturn(rc, rc);
8146
8147	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8148	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8149	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8150	AssertRCReturn(rc, rc);
8151	}
8152
8153	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8154	return rc;
8155	}
8156
8157
8158	/**
8159	* VM-exit exception handler for #DB (Debug exception).
8160	*/
8161	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8162	{
8163	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8164
8165	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8166	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8167	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8168	AssertRCReturn(rc, rc);
8169
8170	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8171	uint64_t uDR6 = X86_DR6_INIT_VAL;
8172	uDR6 \|= (pVmxTransient->uExitQualification
8173	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8174	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8176	if (rc == VINF_EM_RAW_GUEST_TRAP)
8177	{
8178	/** @todo revisit this. */
8179	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8180	pMixedCtx->dr[6] = uDR6;
8181
8182	if (CPUMIsGuestDebugStateActive(pVCpu))
8183	ASMSetDR6(pMixedCtx->dr[6]);
8184
8185	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8186	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8187
8188	/* Paranoia. */
8189	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8190	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8191	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8192
8193	/* Resync DR7. */
8194	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8195
8196	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8197	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8198	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8199	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8200	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8201	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8202	AssertRCReturn(rc,rc);
8203	return rc;
8204	}
8205	/* Return to ring 3 to deal with the debug exit code. */
8206	return rc;
8207	}
8208
8209
8210	/**
8211	* VM-exit exception handler for #NM (Device-not-available exception: floating
8212	* point exception).
8213	*/
8214	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8215	{
8216	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8217
8218	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8219	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8220	#endif
8221
8222	/* We require CR0 and EFER. EFER is always up-to-date. */
8223	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8224	AssertRCReturn(rc, rc);
8225
8226	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8227	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8228	if (rc == VINF_SUCCESS)
8229	{
8230	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8231	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8232	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8233	return VINF_SUCCESS;
8234	}
8235
8236	/* Forward #NM to the guest. */
8237	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8238	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8239	AssertRCReturn(rc, rc);
8240	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8241	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8242	pVmxTransient->cbInstr, 0 /* error code */);
8243	AssertRCReturn(rc, rc);
8244	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8245	return rc;
8246	}
8247
8248
8249	/**
8250	* VM-exit exception handler for #GP (General-protection exception).
8251	*
8252	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8253	*/
8254	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8255	{
8256	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8257
8258	int rc = VERR_INTERNAL_ERROR_5;
8259
8260	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8261	{
8262	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8263	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8264	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8265	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8266	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8267	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8268	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8269	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8270	AssertRCReturn(rc, rc);
8271	return rc;
8272	#else
8273	/* We don't intercept #GP. */
8274	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8275	return VERR_VMX_UNEXPECTED_EXCEPTION;
8276	#endif
8277	}
8278
8279	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8280	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8281
8282	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8283	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8284	AssertRCReturn(rc, rc);
8285
8286	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8287	unsigned int cbOp = 0;
8288	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8289	if (RT_SUCCESS(rc))
8290	{
8291	rc = VINF_SUCCESS;
8292	Assert(cbOp == pDis->cbInstr);
8293	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8294	switch (pDis->pCurInstr->uOpcode)
8295	{
8296	case OP_CLI:
8297	pMixedCtx->eflags.Bits.u1IF = 0;
8298	pMixedCtx->rip += pDis->cbInstr;
8299	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8300	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8301	break;
8302
8303	case OP_STI:
8304	pMixedCtx->eflags.Bits.u1IF = 1;
8305	pMixedCtx->rip += pDis->cbInstr;
8306	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8307	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8308	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE \| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8309	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8310	break;
8311
8312	case OP_HLT:
8313	rc = VINF_EM_HALT;
8314	pMixedCtx->rip += pDis->cbInstr;
8315	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8317	break;
8318
8319	case OP_POPF:
8320	{
8321	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8322	size_t cbParm = 0;
8323	uint32_t uMask = 0;
8324	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8325	{
8326	cbParm = 4;
8327	uMask = 0xffffffff;
8328	}
8329	else
8330	{
8331	cbParm = 2;
8332	uMask = 0xffff;
8333	}
8334
8335	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8336	RTGCPTR GCPtrStack = 0;
8337	X86EFLAGS uEflags;
8338	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8339	&GCPtrStack);
8340	if (RT_SUCCESS(rc))
8341	{
8342	Assert(sizeof(uEflags.u32) >= cbParm);
8343	uEflags.u32 = 0;
8344	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8345	}
8346	if (RT_FAILURE(rc))
8347	{
8348	rc = VERR_EM_INTERPRETER;
8349	break;
8350	}
8351	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8352	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8353	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8354	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8355	pMixedCtx->eflags.Bits.u1RF = 0;
8356	pMixedCtx->esp += cbParm;
8357	pMixedCtx->esp &= uMask;
8358	pMixedCtx->rip += pDis->cbInstr;
8359	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8361	break;
8362	}
8363
8364	case OP_PUSHF:
8365	{
8366	size_t cbParm = 0;
8367	uint32_t uMask = 0;
8368	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8369	{
8370	cbParm = 4;
8371	uMask = 0xffffffff;
8372	}
8373	else
8374	{
8375	cbParm = 2;
8376	uMask = 0xffff;
8377	}
8378
8379	/* Get the stack pointer & push the contents of eflags onto the stack. */
8380	RTGCPTR GCPtrStack = 0;
8381	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8382	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8383	if (RT_FAILURE(rc))
8384	{
8385	rc = VERR_EM_INTERPRETER;
8386	break;
8387	}
8388	X86EFLAGS uEflags;
8389	uEflags = pMixedCtx->eflags;
8390	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8391	uEflags.Bits.u1RF = 0;
8392	uEflags.Bits.u1VM = 0;
8393
8394	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8395	if (RT_FAILURE(rc))
8396	{
8397	rc = VERR_EM_INTERPRETER;
8398	break;
8399	}
8400	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8401	pMixedCtx->esp -= cbParm;
8402	pMixedCtx->esp &= uMask;
8403	pMixedCtx->rip += pDis->cbInstr;
8404	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8405	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8406	break;
8407	}
8408
8409	case OP_IRET:
8410	{
8411	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8412	* instruction reference. */
8413	RTGCPTR GCPtrStack = 0;
8414	uint32_t uMask = 0xffff;
8415	uint16_t aIretFrame[3];
8416	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8417	{
8418	rc = VERR_EM_INTERPRETER;
8419	break;
8420	}
8421	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8422	&GCPtrStack);
8423	if (RT_SUCCESS(rc))
8424	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8425	if (RT_FAILURE(rc))
8426	{
8427	rc = VERR_EM_INTERPRETER;
8428	break;
8429	}
8430	pMixedCtx->eip = 0;
8431	pMixedCtx->ip = aIretFrame[0];
8432	pMixedCtx->cs.Sel = aIretFrame[1];
8433	pMixedCtx->cs.ValidSel = aIretFrame[1];
8434	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8435	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8436	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8437	pMixedCtx->sp += sizeof(aIretFrame);
8438	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8439	\| HM_CHANGED_GUEST_RFLAGS;
8440	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8442	break;
8443	}
8444
8445	case OP_INT:
8446	{
8447	uint16_t uVector = pDis->Param1.uValue & 0xff;
8448	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8449	AssertRCReturn(rc, rc);
8450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8451	break;
8452	}
8453
8454	case OP_INTO:
8455	{
8456	if (pMixedCtx->eflags.Bits.u1OF)
8457	{
8458	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8459	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8460	}
8461	break;
8462	}
8463
8464	default:
8465	{
8466	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8467	EMCODETYPE_SUPERVISOR);
8468	rc = VBOXSTRICTRC_VAL(rc2);
8469	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8470	Log2(("#GP rc=%Rrc\n", rc));
8471	break;
8472	}
8473	}
8474	}
8475	else
8476	rc = VERR_EM_INTERPRETER;
8477
8478	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8479	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8480	("#GP Unexpected rc=%Rrc\n", rc));
8481	return rc;
8482	}
8483
8484
8485	/**
8486	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8487	* the exception reported in the VMX transient structure back into the VM.
8488	*
8489	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8490	* VMX transient structure to be up-to-date.
8491	*/
8492	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8493	{
8494	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8495
8496	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8497	hmR0VmxCheckExitDueToEventDelivery(). */
8498	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8499	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8500	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8501	AssertRCReturn(rc, rc);
8502	return rc;
8503	}
8504
8505
8506	/**
8507	* VM-exit exception handler for #PF (Page-fault exception).
8508	*/
8509	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8510	{
8511	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8512
8513	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8514	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8515	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8516	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8517	AssertRCReturn(rc, rc);
8518
8519	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8520	if (pVM->hm.s.fNestedPaging)
8521	{
8522	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8523	{
8524	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8525	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8526	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8527	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8528	AssertRCReturn(rc, rc);
8529	}
8530	else
8531	{
8532	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8533	Assert(!pVCpu->hm.s.Event.fPending);
8534	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8535	}
8536	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8537	return rc;
8538	}
8539	#else
8540	Assert(!pVM->hm.s.fNestedPaging);
8541	#endif
8542
8543	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8544	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8545	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8546	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8547	AssertRCReturn(rc, rc);
8548	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8549	if ( pVM->hm.s.fTRPPatchingAllowed
8550	&& pVM->hm.s.pGuestPatchMem
8551	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8552	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8553	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8554	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8555	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8556	{
8557	RTGCPHYS GCPhys;
8558	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8559	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8560	if ( rc == VINF_SUCCESS
8561	&& GCPhys == GCPhysApicBase)
8562	{
8563	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8564	AssertRCReturn(rc, rc);
8565
8566	/* Only attempt to patch the instruction once. */
8567	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8568	if (!pPatch)
8569	return VINF_EM_HM_PATCH_TPR_INSTR;
8570	}
8571	}
8572	#endif
8573
8574	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8575	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8576	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8577
8578	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8579	AssertRCReturn(rc, rc);
8580
8581	/* Forward it to the trap handler first. */
8582	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8583	(RTGCPTR)pVmxTransient->uExitQualification);
8584
8585	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8586	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8587
8588	if (rc == VINF_SUCCESS)
8589	{
8590	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8591	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8592	* memory? We don't update the whole state here... */
8593	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8594	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8595
8596	TRPMResetTrap(pVCpu);
8597	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8598	return rc;
8599	}
8600	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8601	{
8602	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8603	{
8604	/* It's a guest page fault and needs to be reflected to the guest. */
8605	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8606	TRPMResetTrap(pVCpu);
8607	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8608	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8609	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8610	pVmxTransient->cbInstr, uGstErrorCode);
8611	AssertRCReturn(rc, rc);
8612	}
8613	else
8614	{
8615	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8616	Assert(!pVCpu->hm.s.Event.fPending);
8617	TRPMResetTrap(pVCpu);
8618	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8619	}
8620	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8621	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8622	return rc;
8623	}
8624
8625	TRPMResetTrap(pVCpu);
8626	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8627	return rc;
8628	}
8629

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45387

Download in other formats: