HMVMXR0.cpp@ 45378

Last change on this file since 45378 was 45352, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 358.0 KB

Line
1	/* $Id: HMVMXR0.cpp 45352 2013-04-04 20:42:26Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
76	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(19)
77	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(21)
79	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
80	\| VMX_UPDATED_GUEST_RIP \
81	\| VMX_UPDATED_GUEST_RSP \
82	\| VMX_UPDATED_GUEST_RFLAGS \
83	\| VMX_UPDATED_GUEST_CR0 \
84	\| VMX_UPDATED_GUEST_CR3 \
85	\| VMX_UPDATED_GUEST_CR4 \
86	\| VMX_UPDATED_GUEST_GDTR \
87	\| VMX_UPDATED_GUEST_IDTR \
88	\| VMX_UPDATED_GUEST_LDTR \
89	\| VMX_UPDATED_GUEST_TR \
90	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
91	\| VMX_UPDATED_GUEST_DEBUG \
92	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
96	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
97	\| VMX_UPDATED_GUEST_INTR_STATE \
98	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
99	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
100	\| VMX_UPDATED_GUEST_APIC_STATE)
101
102	/**
103	* Flags to skip redundant reads of some common VMCS fields.
104	*/
105	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
106	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
107	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
108	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
109	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
110	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
111
112	/*
113	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
114	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
115	* we have Nested Paging support.
116	*/
117	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/* Maximum VM-instruction error number. */
126	#define VMX_INSTR_ERROR_MAX 28
127
128	/*******************************************************************************
129	* Structures and Typedefs *
130	*******************************************************************************/
131	/**
132	* A state structure for holding miscellaneous information across
133	* VMX non-root operation and restored after the transition.
134	*/
135	typedef struct VMXTRANSIENT
136	{
137	/** The host's rflags/eflags. */
138	RTCCUINTREG uEFlags;
139	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
140	uint64_t u64LStarMsr;
141	/** The guest's TPR value used for TPR shadowing. */
142	uint8_t u8GuestTpr;
143
144	/** The basic VM-exit reason. */
145	uint16_t uExitReason;
146	/** The VM-exit exit qualification. */
147	RTGCUINTPTR uExitQualification;
148	/** The VM-exit interruption error code. */
149	uint32_t uExitIntrErrorCode;
150
151	/** The VM-exit interruption-information field. */
152	uint32_t uExitIntrInfo;
153	/** Whether the VM-entry failed or not. */
154	bool fVMEntryFailed;
155	/** The VM-exit instruction-length field. */
156	uint32_t cbInstr;
157
158	/** The VM-entry interruption-information field. */
159	uint32_t uEntryIntrInfo;
160	/** The VM-entry exception error code field. */
161	uint32_t uEntryXcptErrorCode;
162	/** The VM-entry instruction length field. */
163	uint32_t cbEntryInstr;
164
165	/** IDT-vectoring information field. */
166	uint32_t uIdtVectoringInfo;
167	/** IDT-vectoring error code. */
168	uint32_t uIdtVectoringErrorCode;
169
170	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
171	uint32_t fVmcsFieldsRead;
172	/** Whether TSC-offsetting should be setup before VM-entry. */
173	bool fUpdateTscOffsetting;
174	/** Whether the VM-exit was caused by a page-fault during delivery of a
175	* contributary exception or a page-fault. */
176	bool fVectoringPF;
177	} VMXTRANSIENT, *PVMXTRANSIENT;
178
179	/**
180	* MSR-bitmap read permissions.
181	*/
182	typedef enum VMXMSREXITREAD
183	{
184	/** Reading this MSR causes a VM-exit. */
185	VMXMSREXIT_INTERCEPT_READ = 0xb,
186	/** Reading this MSR does not cause a VM-exit. */
187	VMXMSREXIT_PASSTHRU_READ
188	} VMXMSREXITREAD;
189
190	/**
191	* MSR-bitmap write permissions.
192	*/
193	typedef enum VMXMSREXITWRITE
194	{
195	/** Writing to this MSR causes a VM-exit. */
196	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
197	/** Writing to this MSR does not cause a VM-exit. */
198	VMXMSREXIT_PASSTHRU_WRITE
199	} VMXMSREXITWRITE;
200
201	/*******************************************************************************
202	* Internal Functions *
203	*******************************************************************************/
204	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
205	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
206	uint32_t cbInstr, uint32_t u32ErrCode);
207
208	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259
260	/*******************************************************************************
261	* Global Variables *
262	*******************************************************************************/
263	/** @todo Move this to hm_vmx.h. */
264	/**
265	* VM-exit handler.
266	*
267	* @returns VBox status code.
268	* @param pVM Pointer to the VM.
269	* @param pVCpu Pointer to the VMCPU.
270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
271	* out-of-sync. Make sure to update the required
272	* fields before using them.
273	* @param pVmxTransient Pointer to the VMX-transient structure.
274	*/
275	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
278
279	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
280	{
281	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
282	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
283	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
284	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
285	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
286	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
287	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
288	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
289	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
290	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
291	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
292	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
293	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
294	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
295	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
296	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
297	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
298	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
299	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
300	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
301	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
302	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
303	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
304	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
305	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
306	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
307	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
308	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
309	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
310	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
311	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
312	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
313	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
314	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
315	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
316	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
317	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
318	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
319	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
320	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
321	/* 40 UNDEFINED */ hmR0VmxExitPause,
322	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
323	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
324	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
325	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
326	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
327	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
328	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
329	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
330	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
331	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
332	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
333	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
334	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
335	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
336	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
337	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
338	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
339	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
340	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
341	};
342
343	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
344	{
345	/* 0 */ "(Not Used)",
346	/* 1 */ "VMCALL executed in VMX root operation.",
347	/* 2 */ "VMCLEAR with invalid physical address.",
348	/* 3 */ "VMCLEAR with VMXON pointer.",
349	/* 4 */ "VMLAUNCH with non-clear VMCS.",
350	/* 5 */ "VMRESUME with non-launched VMCS.",
351	/* 6 */ "VMRESUME after VMXOFF",
352	/* 7 */ "VM entry with invalid control fields.",
353	/* 8 */ "VM entry with invalid host state fields.",
354	/* 9 */ "VMPTRLD with invalid physical address.",
355	/* 10 */ "VMPTRLD with VMXON pointer.",
356	/* 11 */ "VMPTRLD with incorrect revision identifier.",
357	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
358	/* 13 */ "VMWRITE to read-only VMCS component.",
359	/* 14 */ "(Not Used)",
360	/* 15 */ "VMXON executed in VMX root operation.",
361	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
362	/* 17 */ "VM entry with non-launched executing VMCS.",
363	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
364	/* 19 */ "VMCALL with non-clear VMCS.",
365	/* 20 */ "VMCALL with invalid VM-exit control fields.",
366	/* 21 */ "(Not Used)",
367	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
368	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
369	/* 24 */ "VMCALL with invalid SMM-monitor features.",
370	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
371	/* 26 */ "VM entry with events blocked by MOV SS.",
372	/* 27 */ "(Not Used)",
373	/* 28 */ "Invalid operand to INVEPT/INVVPID."
374	};
375
376
377	/**
378	* Disables longjmps to ring-3.
379	* @param pVCpu Pointer to the VMCPU.
380	*/
381	DECLINLINE(void) hmR0VmxCallRing3Disable(PVMCPU pVCpu)
382	{
383	while (VMMRZCallRing3IsEnabled(pVCpu))
384	VMMRZCallRing3Disable(pVCpu);
385	Assert(VMMR0IsLogFlushDisabled(pVCpu));
386	}
387
388
389	/**
390	* Enables longjmps to ring-3.
391	* @param pVCpu Pointer to the VMCPU.
392	*/
393	DECLINLINE(void) hmR0VmxCallRing3Enable(PVMCPU pVCpu)
394	{
395	while (!VMMRZCallRing3IsEnabled(pVCpu))
396	VMMRZCallRing3Enable(pVCpu);
397	Assert(!VMMR0IsLogFlushDisabled(pVCpu));
398	}
399
400
401	/**
402	* Updates the VM's last error record. If there was a VMX instruction error,
403	* reads the error data from the VMCS and updates VCPU's last error record as
404	* well.
405	*
406	* @param pVM Pointer to the VM.
407	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
408	* VERR_VMX_GENERIC).
409	* @param rc The error code.
410	*/
411	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
412	{
413	AssertPtr(pVM);
414	if (rc == VERR_VMX_GENERIC)
415	{
416	AssertPtrReturnVoid(pVCpu);
417	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
418	}
419	pVM->hm.s.lLastError = rc;
420	}
421
422
423	/**
424	* Reads the VM-entry interruption-information field from the VMCS into the VMX
425	* transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVmxTransient Pointer to the VMX transient structure.
446	*
447	* @remarks No-long-jump zone!!!
448	*/
449	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
450	{
451	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
452	AssertRCReturn(rc, rc);
453	return VINF_SUCCESS;
454	}
455
456
457	/**
458	* Reads the VM-entry exception error code field from the VMCS into
459	* the VMX transient structure.
460	*
461	* @returns VBox status code.
462	* @param pVmxTransient Pointer to the VMX transient structure.
463	*
464	* @remarks No-long-jump zone!!!
465	*/
466	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
467	{
468	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
469	AssertRCReturn(rc, rc);
470	return VINF_SUCCESS;
471	}
472
473
474	/**
475	* Reads the VM-exit interruption-information field from the VMCS into the VMX
476	* transient structure.
477	*
478	* @returns VBox status code.
479	* @param pVmxTransient Pointer to the VMX transient structure.
480	*/
481	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
482	{
483	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
484	{
485	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
486	AssertRCReturn(rc, rc);
487	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
488	}
489	return VINF_SUCCESS;
490	}
491
492
493	/**
494	* Reads the VM-exit interruption error code from the VMCS into the VMX
495	* transient structure.
496	*
497	* @returns VBox status code.
498	* @param pVmxTransient Pointer to the VMX transient structure.
499	*/
500	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
501	{
502	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
503	{
504	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
505	AssertRCReturn(rc, rc);
506	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
507	}
508	return VINF_SUCCESS;
509	}
510
511
512	/**
513	* Reads the VM-exit instruction length field from the VMCS into the VMX
514	* transient structure.
515	*
516	* @returns VBox status code.
517	* @param pVmxTransient Pointer to the VMX transient structure.
518	*/
519	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
520	{
521	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
522	{
523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
524	AssertRCReturn(rc, rc);
525	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
526	}
527	return VINF_SUCCESS;
528	}
529
530
531	/**
532	* Reads the exit qualification from the VMCS into the VMX transient structure.
533	*
534	* @returns VBox status code.
535	* @param pVmxTransient Pointer to the VMX transient structure.
536	*/
537	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
538	{
539	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
540	{
541	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
542	AssertRCReturn(rc, rc);
543	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
544	}
545	return VINF_SUCCESS;
546	}
547
548
549	/**
550	* Reads the IDT-vectoring information field from the VMCS into the VMX
551	* transient structure.
552	*
553	* @returns VBox status code.
554	* @param pVmxTransient Pointer to the VMX transient structure.
555	*
556	* @remarks No-long-jump zone!!!
557	*/
558	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
559	{
560	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
561	{
562	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
563	AssertRCReturn(rc, rc);
564	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
565	}
566	return VINF_SUCCESS;
567	}
568
569
570	/**
571	* Reads the IDT-vectoring error code from the VMCS into the VMX
572	* transient structure.
573	*
574	* @returns VBox status code.
575	* @param pVmxTransient Pointer to the VMX transient structure.
576	*/
577	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
578	{
579	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
580	{
581	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
582	AssertRCReturn(rc, rc);
583	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
584	}
585	return VINF_SUCCESS;
586	}
587
588
589	/**
590	* Enters VMX root mode operation on the current CPU.
591	*
592	* @returns VBox status code.
593	* @param pVM Pointer to the VM (optional, can be NULL, after
594	* a resume).
595	* @param HCPhysCpuPage Physical address of the VMXON region.
596	* @param pvCpuPage Pointer to the VMXON region.
597	*/
598	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
599	{
600	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
601	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
602
603	if (pVM)
604	{
605	/* Write the VMCS revision dword to the VMXON region. */
606	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
607	}
608
609	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
610	RTCCUINTREG fFlags = ASMIntDisableFlags();
611
612	/* Enable the VMX bit in CR4. */
613	RTCCUINTREG uCr4 = ASMGetCR4();
614	if (!(uCr4 & X86_CR4_VMXE))
615	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
616
617	/* Enter VMXON root mode. */
618	int rc = VMXEnable(HCPhysCpuPage);
619
620	/* Restore interrupts. */
621	ASMSetFlags(fFlags);
622	return rc;
623	}
624
625
626	/**
627	* Exits VMX root mode operation on the current CPU.
628	*
629	* @returns VBox status code.
630	*/
631	static int hmR0VmxLeaveRootMode(void)
632	{
633	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
634	RTCCUINTREG fFlags = ASMIntDisableFlags();
635	int rc = VINF_SUCCESS;
636
637	/* If we're for some reason not in VMX root mode, then don't leave it. */
638	if (ASMGetCR4() & X86_CR4_VMXE)
639	{
640	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
641	VMXDisable();
642	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
643	}
644	else
645	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
646
647	/* Restore interrupts. */
648	ASMSetFlags(fFlags);
649	return rc;
650	}
651
652
653	/**
654	* Allocates and maps one physically contiguous page. The allocated page is
655	* zero'd out. (Used by various VT-x structures).
656	*
657	* @returns IPRT status code.
658	* @param pMemObj Pointer to the ring-0 memory object.
659	* @param ppVirt Where to store the virtual address of the
660	* allocation.
661	* @param pPhys Where to store the physical address of the
662	* allocation.
663	*/
664	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
665	{
666	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
667	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
668	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
669
670	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
671	if (RT_FAILURE(rc))
672	return rc;
673	ppVirt = RTR0MemObjAddress(pMemObj);
674	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
675	ASMMemZero32(*ppVirt, PAGE_SIZE);
676	return VINF_SUCCESS;
677	}
678
679
680	/**
681	* Frees and unmaps an allocated physical page.
682	*
683	* @param pMemObj Pointer to the ring-0 memory object.
684	* @param ppVirt Where to re-initialize the virtual address of
685	* allocation as 0.
686	* @param pHCPhys Where to re-initialize the physical address of the
687	* allocation as 0.
688	*/
689	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
690	{
691	AssertPtr(pMemObj);
692	AssertPtr(ppVirt);
693	AssertPtr(pHCPhys);
694	if (*pMemObj != NIL_RTR0MEMOBJ)
695	{
696	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
697	AssertRC(rc);
698	*pMemObj = NIL_RTR0MEMOBJ;
699	*ppVirt = 0;
700	*pHCPhys = 0;
701	}
702	}
703
704
705	/**
706	* Worker function to free VT-x related structures.
707	*
708	* @returns IPRT status code.
709	* @param pVM Pointer to the VM.
710	*/
711	static void hmR0VmxStructsFree(PVM pVM)
712	{
713	for (VMCPUID i = 0; i < pVM->cCpus; i++)
714	{
715	PVMCPU pVCpu = &pVM->aCpus[i];
716	AssertPtr(pVCpu);
717
718	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
720
721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
722	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
723
724	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
726	}
727
728	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
729	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
730	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
731	#endif
732	}
733
734
735	/**
736	* Worker function to allocate VT-x related VM structures.
737	*
738	* @returns IPRT status code.
739	* @param pVM Pointer to the VM.
740	*/
741	static int hmR0VmxStructsAlloc(PVM pVM)
742	{
743	/*
744	* Initialize members up-front so we can cleanup properly on allocation failure.
745	*/
746	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
747	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
748	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
749	pVM->hm.s.vmx.HCPhys##a_Name = 0;
750
751	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
752	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
753	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
754	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
755
756	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
757	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
758	#endif
759	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
760
761	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
762	for (VMCPUID i = 0; i < pVM->cCpus; i++)
763	{
764	PVMCPU pVCpu = &pVM->aCpus[i];
765	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
766	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
767	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
770	}
771	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
772	#undef VMXLOCAL_INIT_VM_MEMOBJ
773
774	/*
775	* Allocate all the VT-x structures.
776	*/
777	int rc = VINF_SUCCESS;
778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
780	if (RT_FAILURE(rc))
781	goto cleanup;
782	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
783	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
784	#endif
785
786	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
787	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
788	{
789	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
790	&pVM->hm.s.vmx.HCPhysApicAccess);
791	if (RT_FAILURE(rc))
792	goto cleanup;
793	}
794
795	/*
796	* Initialize per-VCPU VT-x structures.
797	*/
798	for (VMCPUID i =0; i < pVM->cCpus; i++)
799	{
800	PVMCPU pVCpu = &pVM->aCpus[i];
801	AssertPtr(pVCpu);
802
803	/* Allocate the VM control structure (VMCS). */
804	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
805	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
806	if (RT_FAILURE(rc))
807	goto cleanup;
808
809	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
810	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
811	{
812	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
813	&pVCpu->hm.s.vmx.HCPhysVirtApic);
814	if (RT_FAILURE(rc))
815	goto cleanup;
816	}
817
818	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
820	{
821	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
822	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
826	}
827
828	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
830	if (RT_FAILURE(rc))
831	goto cleanup;
832
833	/* Allocate the VM-exit MSR-load page for the host MSRs. */
834	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	return VINF_SUCCESS;
840
841	cleanup:
842	hmR0VmxStructsFree(pVM);
843	return rc;
844	}
845
846
847	/**
848	* Does global VT-x initialization (called during module initialization).
849	*
850	* @returns VBox status code.
851	*/
852	VMMR0DECL(int) VMXR0GlobalInit(void)
853	{
854	/* Setup the main VM exit handlers. */
855	/** @todo I'm told lookup table for function pointers is a bad idea in terms
856	* of performance. Make these static for now and not do change anything
857	* at runtime so we can easily switch to a switch-case approach later. */
858	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
859	#ifdef DEBUG
860	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
861	Assert(s_apfnVMExitHandlers[i]);
862	#endif
863	return VINF_SUCCESS;
864	}
865
866
867	/**
868	* Does global VT-x termination (called during module termination).
869	*/
870	VMMR0DECL(void) VMXR0GlobalTerm()
871	{
872	/* Nothing to do currently. */
873	}
874
875
876	/**
877	* Sets up and activates VT-x on the current CPU.
878	*
879	* @returns VBox status code.
880	* @param pCpu Pointer to the global CPU info struct.
881	* @param pVM Pointer to the VM (can be NULL after a host resume
882	* operation).
883	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
884	* fEnabledByHost is true).
885	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
886	* @a fEnabledByHost is true).
887	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
888	* enable VT-x/AMD-V on the host.
889	*/
890	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
891	{
892	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
893	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
894
895	if (!fEnabledByHost)
896	{
897	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
898	if (RT_FAILURE(rc))
899	return rc;
900	}
901
902	/*
903	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
904	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
905	* each time while reusing a VPID after hitting the MaxASID limit once.
906	*/
907	if ( pVM
908	&& pVM->hm.s.vmx.fVpid
909	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
910	{
911	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
912	pCpu->fFlushAsidBeforeUse = false;
913	}
914	else
915	pCpu->fFlushAsidBeforeUse = true;
916
917	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
918	++pCpu->cTlbFlushes;
919
920	return VINF_SUCCESS;
921	}
922
923
924	/**
925	* Deactivates VT-x on the current CPU.
926	*
927	* @returns VBox status code.
928	* @param pCpu Pointer to the global CPU info struct.
929	* @param pvCpuPage Pointer to the VMXON region.
930	* @param HCPhysCpuPage Physical address of the VMXON region.
931	*/
932	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
933	{
934	NOREF(pCpu);
935	NOREF(pvCpuPage);
936	NOREF(HCPhysCpuPage);
937
938	hmR0VmxLeaveRootMode();
939	return VINF_SUCCESS;
940	}
941
942
943	/**
944	* Sets the permission bits for the specified MSR in the MSR bitmap.
945	*
946	* @param pVCpu Pointer to the VMCPU.
947	* @param ulMSR The MSR value.
948	* @param enmRead Whether reading this MSR causes a VM-exit.
949	* @param enmWrite Whether writing this MSR causes a VM-exit.
950	*/
951	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
952	{
953	unsigned ulBit;
954	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
955
956	/*
957	* Layout:
958	* 0x000 - 0x3ff - Low MSR read bits
959	* 0x400 - 0x7ff - High MSR read bits
960	* 0x800 - 0xbff - Low MSR write bits
961	* 0xc00 - 0xfff - High MSR write bits
962	*/
963	if (ulMsr <= 0x00001FFF)
964	{
965	/* Pentium-compatible MSRs */
966	ulBit = ulMsr;
967	}
968	else if ( ulMsr >= 0xC0000000
969	&& ulMsr <= 0xC0001FFF)
970	{
971	/* AMD Sixth Generation x86 Processor MSRs */
972	ulBit = (ulMsr - 0xC0000000);
973	pbMsrBitmap += 0x400;
974	}
975	else
976	{
977	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
978	return;
979	}
980
981	Assert(ulBit <= 0x1fff);
982	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
983	ASMBitSet(pbMsrBitmap, ulBit);
984	else
985	ASMBitClear(pbMsrBitmap, ulBit);
986
987	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
988	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
989	else
990	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
991	}
992
993
994	/**
995	* Flushes the TLB using EPT.
996	*
997	* @returns VBox status code.
998	* @param pVM Pointer to the VM.
999	* @param pVCpu Pointer to the VMCPU.
1000	* @param enmFlush Type of flush.
1001	*/
1002	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1003	{
1004	AssertPtr(pVM);
1005	Assert(pVM->hm.s.fNestedPaging);
1006
1007	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1008
1009	uint64_t descriptor[2];
1010	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
1011	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1012
1013	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1014	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1015	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1016	}
1017
1018
1019	/**
1020	* Flushes the TLB using VPID.
1021	*
1022	* @returns VBox status code.
1023	* @param pVM Pointer to the VM.
1024	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1025	* enmFlush).
1026	* @param enmFlush Type of flush.
1027	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1028	* on @a enmFlush).
1029	*/
1030	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.vmx.fVpid);
1034
1035	uint64_t descriptor[2];
1036	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1037	{
1038	descriptor[0] = 0;
1039	descriptor[1] = 0;
1040	}
1041	else
1042	{
1043	AssertPtr(pVCpu);
1044	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1045	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1046	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1047	descriptor[1] = GCPtr;
1048	}
1049
1050	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1051	AssertMsg(rc == VINF_SUCCESS,
1052	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1053	if ( RT_SUCCESS(rc)
1054	&& pVCpu)
1055	{
1056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1057	}
1058	}
1059
1060
1061	/**
1062	* Invalidates a guest page by guest virtual address. Only relevant for
1063	* EPT/VPID, otherwise there is nothing really to invalidate.
1064	*
1065	* @returns VBox status code.
1066	* @param pVM Pointer to the VM.
1067	* @param pVCpu Pointer to the VMCPU.
1068	* @param GCVirt Guest virtual address of the page to invalidate.
1069	*/
1070	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1071	{
1072	AssertPtr(pVM);
1073	AssertPtr(pVCpu);
1074	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1075
1076	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1077	if (!fFlushPending)
1078	{
1079	/*
1080	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1081	* See @bugref{6043} and @bugref{6177}.
1082	*
1083	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1084	* function maybe called in a loop with individual addresses.
1085	*/
1086	if (pVM->hm.s.vmx.fVpid)
1087	{
1088	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1089	{
1090	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1091	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1092	}
1093	else
1094	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1095	}
1096	else if (pVM->hm.s.fNestedPaging)
1097	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1098	}
1099
1100	return VINF_SUCCESS;
1101	}
1102
1103
1104	/**
1105	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1106	* otherwise there is nothing really to invalidate.
1107	*
1108	* @returns VBox status code.
1109	* @param pVM Pointer to the VM.
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param GCPhys Guest physical address of the page to invalidate.
1112	*/
1113	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1114	{
1115	LogFlowFunc(("%RGp\n", GCPhys));
1116
1117	/*
1118	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1119	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1120	* This function might be called in a loop.
1121	*/
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1130	* case where neither EPT nor VPID is supported by the CPU.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	*
1135	* @remarks Called with interrupts disabled.
1136	*/
1137	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1138	{
1139	NOREF(pVM);
1140	AssertPtr(pVCpu);
1141	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1143
1144	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1145	AssertPtr(pCpu);
1146
1147	pVCpu->hm.s.TlbShootdown.cPages = 0;
1148	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1149	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1150	pVCpu->hm.s.fForceTLBFlush = false;
1151	return;
1152	}
1153
1154
1155	/**
1156	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1157	*
1158	* @param pVM Pointer to the VM.
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @remarks All references to "ASID" in this function pertains to "VPID" in
1161	* Intel's nomenclature. The reason is, to avoid confusion in compare
1162	* statements since the host-CPU copies are named "ASID".
1163	*
1164	* @remarks Called with interrupts disabled.
1165	* @todo Statistics.
1166	*/
1167	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1168	{
1169	AssertPtr(pVM);
1170	AssertPtr(pVCpu);
1171	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1172	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1173	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1174
1175	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1176	AssertPtr(pCpu);
1177
1178	/*
1179	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1180	* This can happen both for start & resume due to long jumps back to ring-3.
1181	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1182	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1183	*/
1184	bool fNewASID = false;
1185	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1186	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1187	{
1188	pVCpu->hm.s.fForceTLBFlush = true;
1189	fNewASID = true;
1190	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1191	}
1192
1193	/*
1194	* Check for explicit TLB shootdowns.
1195	*/
1196	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1197	{
1198	pVCpu->hm.s.fForceTLBFlush = true;
1199	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1200	}
1201
1202	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1203	if (pVCpu->hm.s.fForceTLBFlush)
1204	{
1205	if (fNewASID)
1206	{
1207	++pCpu->uCurrentAsid;
1208	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1209	{
1210	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1211	pCpu->cTlbFlushes++;
1212	pCpu->fFlushAsidBeforeUse = true;
1213	}
1214
1215	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1216	if (pCpu->fFlushAsidBeforeUse)
1217	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1218	}
1219	else
1220	{
1221	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1222	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1223	else
1224	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1225	}
1226
1227	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1228	pVCpu->hm.s.fForceTLBFlush = false;
1229	}
1230	else
1231	{
1232	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1233	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1234	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1235	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1236
1237	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1238	* not be executed. See hmQueueInvlPage() where it is commented
1239	* out. Support individual entry flushing someday. */
1240	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1241	{
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1243
1244	/*
1245	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1246	* as supported by the CPU.
1247	*/
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1249	{
1250	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1251	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1252	}
1253	else
1254	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1255	}
1256	else
1257	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1258	}
1259	pVCpu->hm.s.TlbShootdown.cPages = 0;
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1263	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1264	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1265	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1266	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1267	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1268
1269	/* Update VMCS with the VPID. */
1270	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1271	AssertRC(rc);
1272	}
1273
1274
1275	/**
1276	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1277	*
1278	* @returns VBox status code.
1279	* @param pVM Pointer to the VM.
1280	* @param pVCpu Pointer to the VMCPU.
1281	*
1282	* @remarks Called with interrupts disabled.
1283	* @todo Statistics.
1284	*/
1285	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1286	{
1287	AssertPtr(pVM);
1288	AssertPtr(pVCpu);
1289	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1290	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1291
1292	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1293	AssertPtr(pCpu);
1294
1295	/*
1296	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1297	* This can happen both for start & resume due to long jumps back to ring-3.
1298	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1299	*/
1300	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1301	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1302	{
1303	pVCpu->hm.s.fForceTLBFlush = true;
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1305	}
1306
1307	/* Check for explicit TLB shootdown flushes. */
1308	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1309	{
1310	pVCpu->hm.s.fForceTLBFlush = true;
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1312	}
1313
1314	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1315	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1316
1317	if (pVCpu->hm.s.fForceTLBFlush)
1318	{
1319	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1320	pVCpu->hm.s.fForceTLBFlush = false;
1321	}
1322	else
1323	{
1324	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1325	* not be executed. See hmQueueInvlPage() where it is commented
1326	* out. Support individual entry flushing someday. */
1327	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1328	{
1329	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	}
1333	else
1334	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1335	}
1336
1337	pVCpu->hm.s.TlbShootdown.cPages= 0;
1338	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1339	}
1340
1341
1342	/**
1343	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1344	*
1345	* @returns VBox status code.
1346	* @param pVM Pointer to the VM.
1347	* @param pVCpu Pointer to the VMCPU.
1348	*
1349	* @remarks Called with interrupts disabled.
1350	* @todo Statistics.
1351	*/
1352	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1353	{
1354	AssertPtr(pVM);
1355	AssertPtr(pVCpu);
1356	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1357	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1358
1359	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1360
1361	/*
1362	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1363	* This can happen both for start & resume due to long jumps back to ring-3.
1364	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1365	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1366	*/
1367	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1368	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1369	{
1370	pVCpu->hm.s.fForceTLBFlush = true;
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1372	}
1373
1374	/* Check for explicit TLB shootdown flushes. */
1375	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1376	{
1377	/*
1378	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1379	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1380	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1381	*/
1382	pVCpu->hm.s.fForceTLBFlush = true;
1383	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1384	}
1385
1386	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1387	if (pVCpu->hm.s.fForceTLBFlush)
1388	{
1389	++pCpu->uCurrentAsid;
1390	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1391	{
1392	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1393	pCpu->fFlushAsidBeforeUse = true;
1394	pCpu->cTlbFlushes++;
1395	}
1396
1397	pVCpu->hm.s.fForceTLBFlush = false;
1398	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1399	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1400	if (pCpu->fFlushAsidBeforeUse)
1401	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1402	}
1403	else
1404	{
1405	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1406	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1407	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1408	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1409
1410	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1411	* not be executed. See hmQueueInvlPage() where it is commented
1412	* out. Support individual entry flushing someday. */
1413	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1414	{
1415	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1416	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1417	{
1418	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1419	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1420	}
1421	else
1422	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1423	}
1424	else
1425	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1426	}
1427
1428	pVCpu->hm.s.TlbShootdown.cPages = 0;
1429	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1430
1431	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1432	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1433	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1434	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1435	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1436	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1437
1438	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1439	AssertRC(rc);
1440	}
1441
1442
1443	/**
1444	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1445	* TLB entries from the host TLB before VM-entry.
1446	*
1447	* @returns VBox status code.
1448	* @param pVM Pointer to the VM.
1449	*/
1450	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1451	{
1452	/*
1453	* Determine optimal flush type for nested paging.
1454	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1455	* guest execution (see hmR3InitFinalizeR0()).
1456	*/
1457	if (pVM->hm.s.fNestedPaging)
1458	{
1459	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1460	{
1461	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1462	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1463	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1464	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1465	else
1466	{
1467	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1468	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1469	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1470	}
1471
1472	/* Make sure the write-back cacheable memory type for EPT is supported. */
1473	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1474	{
1475	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1476	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1477	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1478	}
1479	}
1480	else
1481	{
1482	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1483	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1484	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1485	}
1486	}
1487
1488	/*
1489	* Determine optimal flush type for VPID.
1490	*/
1491	if (pVM->hm.s.vmx.fVpid)
1492	{
1493	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1494	{
1495	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1496	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1497	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1499	else
1500	{
1501	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1502	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1503	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1504	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1505	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1506	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1507	pVM->hm.s.vmx.fVpid = false;
1508	}
1509	}
1510	else
1511	{
1512	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1513	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1515	pVM->hm.s.vmx.fVpid = false;
1516	}
1517	}
1518
1519	/*
1520	* Setup the handler for flushing tagged-TLBs.
1521	*/
1522	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1523	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1524	else if (pVM->hm.s.fNestedPaging)
1525	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1526	else if (pVM->hm.s.vmx.fVpid)
1527	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1528	else
1529	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1530	return VINF_SUCCESS;
1531	}
1532
1533
1534	/**
1535	* Sets up pin-based VM-execution controls in the VMCS.
1536	*
1537	* @returns VBox status code.
1538	* @param pVM Pointer to the VM.
1539	* @param pVCpu Pointer to the VMCPU.
1540	*/
1541	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1542	{
1543	AssertPtr(pVM);
1544	AssertPtr(pVCpu);
1545
1546	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1547	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1548
1549	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1550	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1551	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1552
1553	/* Enable the VMX preemption timer. */
1554	if (pVM->hm.s.vmx.fUsePreemptTimer)
1555	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1556
1557	if ((val & zap) != val)
1558	{
1559	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1560	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1562	}
1563
1564	val &= zap;
1565	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1566	AssertRCReturn(rc, rc);
1567
1568	/* Update VCPU with the currently set pin-based VM-execution controls. */
1569	pVCpu->hm.s.vmx.u32PinCtls = val;
1570	return rc;
1571	}
1572
1573
1574	/**
1575	* Sets up processor-based VM-execution controls in the VMCS.
1576	*
1577	* @returns VBox status code.
1578	* @param pVM Pointer to the VM.
1579	* @param pVMCPU Pointer to the VMCPU.
1580	*/
1581	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1582	{
1583	AssertPtr(pVM);
1584	AssertPtr(pVCpu);
1585
1586	int rc = VERR_INTERNAL_ERROR_5;
1587	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1588	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1589
1590	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1591	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1592	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1593	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1594	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1595	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1596	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1597
1598	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1599	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1600	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1601	{
1602	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1603	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1604	}
1605
1606	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1607	if (!pVM->hm.s.fNestedPaging)
1608	{
1609	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1613	}
1614
1615	/* Use TPR shadowing if supported by the CPU. */
1616	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1617	{
1618	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1619	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1620	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1622
1623	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1624	/* CR8 writes causes a VM-exit based on TPR threshold. */
1625	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1626	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1627	}
1628	else
1629	{
1630	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1632	}
1633
1634	/* Use MSR-bitmaps if supported by the CPU. */
1635	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1636	{
1637	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1638
1639	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1640	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1641	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1642	AssertRCReturn(rc, rc);
1643
1644	/*
1645	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1646	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1647	*/
1648	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1649	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1650	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1651	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1652	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1653	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1654	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1655	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1656	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1657	}
1658
1659	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1660	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1662
1663	if ((val & zap) != val)
1664	{
1665	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1666	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1667	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1668	}
1669
1670	val &= zap;
1671	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1672	AssertRCReturn(rc, rc);
1673
1674	/* Update VCPU with the currently set processor-based VM-execution controls. */
1675	pVCpu->hm.s.vmx.u32ProcCtls = val;
1676
1677	/*
1678	* Secondary processor-based VM-execution controls.
1679	*/
1680	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681	{
1682	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1683	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1684
1685	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1686
1687	if (pVM->hm.s.fNestedPaging)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1689	else
1690	{
1691	/*
1692	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1693	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1694	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1695	*/
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1698	}
1699
1700	if (pVM->hm.s.vmx.fVpid)
1701	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1702
1703	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1704	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1705
1706	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1707	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1708	* done dynamically. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1710	{
1711	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1712	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1713	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1714	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1715	AssertRCReturn(rc, rc);
1716	}
1717
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1719	{
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1722	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1723	}
1724
1725	if ((val & zap) != val)
1726	{
1727	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1728	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1729	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1730	}
1731
1732	val &= zap;
1733	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1734	AssertRCReturn(rc, rc);
1735
1736	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1737	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1738	}
1739
1740	return VINF_SUCCESS;
1741	}
1742
1743
1744	/**
1745	* Sets up miscellaneous (everything other than Pin & Processor-based
1746	* VM-execution) control fields in the VMCS.
1747	*
1748	* @returns VBox status code.
1749	* @param pVM Pointer to the VM.
1750	* @param pVCpu Pointer to the VMCPU.
1751	*/
1752	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1753	{
1754	AssertPtr(pVM);
1755	AssertPtr(pVCpu);
1756
1757	int rc = VERR_GENERAL_FAILURE;
1758
1759	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1760	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1761
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1763
1764	/*
1765	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1766	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1767	* We thus use the exception bitmap to control it rather than use both.
1768	*/
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1770	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1771
1772	/** @todo Explore possibility of using IO-bitmaps. */
1773	/* All IO & IOIO instructions cause VM-exits. */
1774	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1775	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1776
1777	/* Setup MSR autoloading/autostoring. */
1778	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1779	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1781	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1782	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1783	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1784
1785	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1786	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1787	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1788	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1789
1790	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1791	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1792	AssertRCReturn(rc, rc);
1793	return rc;
1794	}
1795
1796
1797	/**
1798	* Sets up the initial exception bitmap in the VMCS based on static conditions
1799	* (i.e. conditions that cannot ever change at runtime).
1800	*
1801	* @returns VBox status code.
1802	* @param pVM Pointer to the VM.
1803	* @param pVCpu Pointer to the VMCPU.
1804	*/
1805	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1806	{
1807	AssertPtr(pVM);
1808	AssertPtr(pVCpu);
1809
1810	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1811
1812	uint32_t u32XcptBitmap = 0;
1813
1814	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1815	if (!pVM->hm.s.fNestedPaging)
1816	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1817
1818	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1819	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1820	AssertRCReturn(rc, rc);
1821	return rc;
1822	}
1823
1824
1825	/**
1826	* Sets up the initial guest-state mask. The guest-state mask is consulted
1827	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1828	* for the nested virtualization case (as it would cause a VM-exit).
1829	*
1830	* @param pVCpu Pointer to the VMCPU.
1831	*/
1832	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1833	{
1834	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1835	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1836	return VINF_SUCCESS;
1837	}
1838
1839
1840	/**
1841	* Does per-VM VT-x initialization.
1842	*
1843	* @returns VBox status code.
1844	* @param pVM Pointer to the VM.
1845	*/
1846	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1847	{
1848	LogFlowFunc(("pVM=%p\n", pVM));
1849
1850	int rc = hmR0VmxStructsAlloc(pVM);
1851	if (RT_FAILURE(rc))
1852	{
1853	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1854	return rc;
1855	}
1856
1857	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1858	{
1859	PVMCPU pVCpu = &pVM->aCpus[i];
1860
1861	/* Current guest paging mode. */
1862	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1863	}
1864
1865	return VINF_SUCCESS;
1866	}
1867
1868
1869	/**
1870	* Does per-VM VT-x termination.
1871	*
1872	* @returns VBox status code.
1873	* @param pVM Pointer to the VM.
1874	*/
1875	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1876	{
1877	LogFlowFunc(("pVM=%p\n", pVM));
1878
1879	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1880	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1881	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1882	#endif
1883	hmR0VmxStructsFree(pVM);
1884	return VINF_SUCCESS;
1885	}
1886
1887
1888	/**
1889	* Sets up the VM for execution under VT-x.
1890	* This function is only called once per-VM during initalization.
1891	*
1892	* @returns VBox status code.
1893	* @param pVM Pointer to the VM.
1894	*/
1895	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1896	{
1897	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1898	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1899
1900	LogFlowFunc(("pVM=%p\n", pVM));
1901
1902	/*
1903	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1904	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1905	*/
1906	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1907	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1908	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1909	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1910	{
1911	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1912	return VERR_INTERNAL_ERROR;
1913	}
1914
1915	/* Initialize these always, see hmR3InitFinalizeR0().*/
1916	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1917	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1918
1919	/* Setup the tagged-TLB flush handlers. */
1920	int rc = hmR0VmxSetupTaggedTlb(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1928	{
1929	PVMCPU pVCpu = &pVM->aCpus[i];
1930	AssertPtr(pVCpu);
1931	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1932
1933	/* Set revision dword at the beginning of the VMCS structure. */
1934	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1935
1936	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1937	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Load this VMCS as the current VMCS. */
1942	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the pin-based VM-execution controls. */
1947	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	/* Setup the processor-based VM-execution controls. */
1952	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1953	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1954	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1955
1956	/* Setup the rest (miscellaneous) VM-execution controls. */
1957	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	/* Setup the initial exception bitmap. */
1962	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1963	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1964	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1965
1966	/* Setup the initial guest-state mask. */
1967	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1968	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1969	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1970
1971	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1972	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1973	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1974	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1975	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1976	#endif
1977
1978	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1979	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1980	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1981	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1982
1983	/* Update the last error record for this VCPU. */
1984	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1985	}
1986
1987	return VINF_SUCCESS;
1988	}
1989
1990
1991	/**
1992	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1993	* the VMCS.
1994	*
1995	* @returns VBox status code.
1996	* @param pVM Pointer to the VM.
1997	* @param pVCpu Pointer to the VMCPU.
1998	*/
1999	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2000	{
2001	RTCCUINTREG uReg = ASMGetCR0();
2002	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2003	AssertRCReturn(rc, rc);
2004
2005	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2006	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2007	if (VMX_IS_64BIT_HOST_MODE())
2008	{
2009	uint64_t uReg = hmR0Get64bitCR3();
2010	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2011	}
2012	else
2013	#endif
2014	{
2015	uReg = ASMGetCR3();
2016	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2017	}
2018	AssertRCReturn(rc, rc);
2019
2020	uReg = ASMGetCR4();
2021	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2022	AssertRCReturn(rc, rc);
2023	return rc;
2024	}
2025
2026
2027	/**
2028	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2029	* the host-state area in the VMCS.
2030	*
2031	* @returns VBox status code.
2032	* @param pVM Pointer to the VM.
2033	* @param pVCpu Pointer to the VMCPU.
2034	*/
2035	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2036	{
2037	int rc = VERR_INTERNAL_ERROR_5;
2038	RTSEL uSelCS = 0;
2039	RTSEL uSelSS = 0;
2040	RTSEL uSelDS = 0;
2041	RTSEL uSelES = 0;
2042	RTSEL uSelFS = 0;
2043	RTSEL uSelGS = 0;
2044	RTSEL uSelTR = 0;
2045
2046	/*
2047	* Host Selector registers.
2048	*/
2049	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2050	if (VMX_IS_64BIT_HOST_MODE())
2051	{
2052	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2053	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2054	}
2055	else
2056	{
2057	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2058	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2059	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2060	}
2061	#else
2062	uSelCS = ASMGetCS();
2063	uSelSS = ASMGetSS();
2064	#endif
2065
2066	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2067	/** @todo Verify if we have any platform that actually run with DS or ES with
2068	* RPL != 0 in kernel space. */
2069	uSelDS = 0;
2070	uSelES = 0;
2071	uSelFS = 0;
2072	uSelGS = 0;
2073	uSelTR = ASMGetTR();
2074
2075	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2076	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2077	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2078	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2079	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2080	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2081	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2082	Assert(uSelCS != 0);
2083	Assert(uSelTR != 0);
2084
2085	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2086	#if 0
2087	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2088	Assert(uSelSS != 0);
2089	#endif
2090
2091	/* Write these host selector fields into the host-state area in the VMCS. */
2092	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2093	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2094	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2095	#if 0
2096	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2097	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2098	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2099	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2100	#endif
2101	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2102	AssertRCReturn(rc, rc);
2103
2104	/*
2105	* Host GDTR and IDTR.
2106	*/
2107	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2108	* be safe to -not- save and restore GDTR and IDTR in the assembly
2109	* code and just do it here and don't care if the limits are zapped on
2110	* VM-exit. */
2111	RTGDTR Gdtr;
2112	RT_ZERO(Gdtr);
2113	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2114	if (VMX_IS_64BIT_HOST_MODE())
2115	{
2116	X86XDTR64 Gtr64;
2117	X86XDTR64 Idtr64;
2118	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2119	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2120	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2121	Gdtr.cbGdt = Gdtr64.cb;
2122	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2123	}
2124	else
2125	#endif
2126	{
2127	RTIDTR Idtr;
2128	ASMGetGDTR(&Gdtr);
2129	ASMGetIDTR(&Idtr);
2130	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2131	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2132	}
2133	AssertRCReturn(rc, rc);
2134
2135	/*
2136	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2137	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2138	*/
2139	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2140	{
2141	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2142	return VERR_VMX_INVALID_HOST_STATE;
2143	}
2144
2145	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2146	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2147	if (VMX_IS_64BIT_HOST_MODE())
2148	{
2149	/* We need the 64-bit TR base for hybrid darwin. */
2150	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2151	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2152	}
2153	else
2154	#endif
2155	{
2156	uintptr_t uTRBase;
2157	#if HC_ARCH_BITS == 64
2158	uTRBase = X86DESC64_BASE(pDesc);
2159	#else
2160	uTRBase = X86DESC_BASE(pDesc);
2161	#endif
2162	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2163	}
2164	AssertRCReturn(rc, rc);
2165
2166	/*
2167	* Host FS base and GS base.
2168	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2169	* would take care of the bases. In 64-bit, the MSRs come into play.
2170	*/
2171	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2172	if (VMX_IS_64BIT_HOST_MODE())
2173	{
2174	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2175	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2176	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2177	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2178	AssertRCReturn(rc, rc);
2179	}
2180	#endif
2181	return rc;
2182	}
2183
2184
2185	/**
2186	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2187	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2188	* the host after every successful VM exit.
2189	*
2190	* @returns VBox status code.
2191	* @param pVM Pointer to the VM.
2192	* @param pVCpu Pointer to the VMCPU.
2193	*/
2194	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2195	{
2196	AssertPtr(pVCpu);
2197	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2198
2199	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2200	unsigned idxHostMsr = 0;
2201	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2202
2203	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2204	{
2205	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2206	pHostMsr->u32Reserved = 0;
2207	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2208	if (CPUMIsGuestInLongMode(pVCpu))
2209	{
2210	/* Must match the EFER value in our 64 bits switcher. */
2211	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2212	}
2213	else
2214	#endif
2215	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2216	pHostMsr++; idxHostMsr++;
2217	}
2218
2219	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2220	if (VMX_IS_64BIT_HOST_MODE())
2221	{
2222	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2223	pHostMsr->u32Reserved = 0;
2224	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2225	pHostMsr++; idxHostMsr++;
2226	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2227	pHostMsr->u32Reserved = 0;
2228	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2229	pHostMsr++; idxHostMsr++;
2230	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2231	pHostMsr->u32Reserved = 0;
2232	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2233	pHostMsr++; idxHostMsr++;
2234	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2235	#if 0
2236	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2237	pMsr->u32Reserved = 0;
2238	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2239	pHostMsr++; idxHostMsr++;
2240	#endif
2241	}
2242	#endif
2243
2244	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2245	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2246	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2247
2248	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2249	AssertRCReturn(rc, rc);
2250
2251	/*
2252	* Host Sysenter MSRs.
2253	*/
2254	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2255	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2256	if (VMX_IS_64BIT_HOST_MODE())
2257	{
2258	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2259	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2260	}
2261	else
2262	{
2263	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2264	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2265	}
2266	#elif HC_ARCH_BITS == 32
2267	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2268	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2269	#else
2270	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2271	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2272	#endif
2273	AssertRCReturn(rc, rc);
2274
2275	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2276	* hmR0VmxSetupExitCtls() !! */
2277	return rc;
2278	}
2279
2280
2281	/**
2282	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2283	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2284	* controls".
2285	*
2286	* @returns VBox status code.
2287	* @param pVM Pointer to the VM.
2288	* @param pVCpu Pointer to the VMCPU.
2289	* @param pCtx Pointer to the guest-CPU context.
2290	*
2291	* @remarks No-long-jump zone!!!
2292	*/
2293	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2294	{
2295	int rc = VINF_SUCCESS;
2296	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2297	{
2298	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2299	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2300
2301	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2302	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2303
2304	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2305	if (CPUMIsGuestInLongModeEx(pCtx))
2306	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2307	else
2308	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2309
2310	/*
2311	* The following should not be set (since we're not in SMM mode):
2312	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2313	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2314	*/
2315
2316	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2317	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2318	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2319
2320	if ((val & zap) != val)
2321	{
2322	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2323	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2324	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2325	}
2326
2327	val &= zap;
2328	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2329	AssertRCReturn(rc, rc);
2330
2331	/* Update VCPU with the currently set VM-exit controls. */
2332	pVCpu->hm.s.vmx.u32EntryCtls = val;
2333	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2334	}
2335	return rc;
2336	}
2337
2338
2339	/**
2340	* Sets up the VM-exit controls in the VMCS.
2341	*
2342	* @returns VBox status code.
2343	* @param pVM Pointer to the VM.
2344	* @param pVCpu Pointer to the VMCPU.
2345	*/
2346	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2347	{
2348	int rc = VINF_SUCCESS;
2349	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2350	{
2351	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2355	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2356
2357	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2358	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2359	if (VMX_IS_64BIT_HOST_MODE())
2360	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2361	else
2362	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2363	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2364	if (CPUMIsGuestInLongModeEx(pCtx))
2365	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2366	else
2367	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2368	#endif
2369
2370	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2371	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2372
2373	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2374	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2375	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2376	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2377	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR,
2378	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER. */
2379
2380	if ((val & zap) != val)
2381	{
2382	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2383	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2385	}
2386
2387	val &= zap;
2388	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2389
2390	/* Update VCPU with the currently set VM-exit controls. */
2391	pVCpu->hm.s.vmx.u32ExitCtls = val;
2392	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2393	}
2394	return rc;
2395	}
2396
2397
2398	/**
2399	* Loads the guest APIC and related state.
2400	*
2401	* @returns VBox status code.
2402	* @param pVM Pointer to the VM.
2403	* @param pVCpu Pointer to the VMCPU.
2404	* @param pCtx Pointer to the guest-CPU context.
2405	*/
2406	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2407	{
2408	int rc = VINF_SUCCESS;
2409	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2410	{
2411	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2412	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2413	{
2414	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2415
2416	bool fPendingIntr = false;
2417	uint8_t u8GuestTpr = 0;
2418	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2419	AssertRCReturn(rc, rc);
2420
2421	/*
2422	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2423	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2424	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2425	* the interrupt when we VM-exit for other reasons.
2426	*/
2427	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2428	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2429	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2430	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2431
2432	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2433	AssertRCReturn(rc, rc);
2434
2435	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2436	if (pVM->hm.s.fTPRPatchingActive)
2437	{
2438	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2439	pCtx->msrLSTAR = u8GuestTpr;
2440	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2441	{
2442	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2443	if (fPendingIntr)
2444	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2445	else
2446	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447	}
2448	}
2449	}
2450
2451	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2452	}
2453	return rc;
2454	}
2455
2456
2457	/**
2458	* Loads the guest's RIP into the guest-state area in the VMCS.
2459	*
2460	* @returns VBox status code.
2461	* @param pVM Pointer to the VM.
2462	* @param pVCpu Pointer to the VMCPU.
2463	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2464	* out-of-sync. Make sure to update the required fields
2465	* before using them.
2466	*
2467	* @remarks No-long-jump zone!!!
2468	*/
2469	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2470	{
2471	int rc = VINF_SUCCESS;
2472	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2473	{
2474	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2475	AssertRCReturn(rc, rc);
2476	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2477	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2478	}
2479	return rc;
2480	}
2481
2482
2483	/**
2484	* Loads the guest's RSP into the guest-state area in the VMCS.
2485	*
2486	* @returns VBox status code.
2487	* @param pVM Pointer to the VM.
2488	* @param pVCpu Pointer to the VMCPU.
2489	* @param pCtx Pointer to the guest-CPU context.
2490	*
2491	* @remarks No-long-jump zone!!!
2492	*/
2493	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2494	{
2495	int rc = VINF_SUCCESS;
2496	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2497	{
2498	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2499	AssertRCReturn(rc, rc);
2500	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2501	}
2502	return rc;
2503	}
2504
2505
2506	/**
2507	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	* @param pCtx Pointer to the guest-CPU context.
2513	*
2514	* @remarks No-long-jump zone!!!
2515	*/
2516	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2517	{
2518	int rc = VINF_SUCCESS;
2519	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2520	{
2521	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2522	Let us assert it as such and use native-width VMWRITE. */
2523	X86RFLAGS uRFlags = pCtx->rflags;
2524	Assert(uRFlags.u64 >> 32 == 0);
2525	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2526	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2527
2528	/*
2529	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2530	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2531	*/
2532	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2533	{
2534	Assert(pVM->hm.s.vmx.pRealModeTSS);
2535	Assert(PDMVmmDevHeapIsEnabled(pVM));
2536	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2537	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2538	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2539	}
2540
2541	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2542	AssertRCReturn(rc, rc);
2543
2544	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2545	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2546	}
2547	return rc;
2548	}
2549
2550
2551	/**
2552	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2553	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2554	* assembly code.
2555	*
2556	* @returns VBox status code.
2557	* @param pVM Pointer to the VM.
2558	* @param pVCpu Pointer to the VMCPU.
2559	* @param pCtx Pointer to the guest-CPU context.
2560	*
2561	* @remarks No-long-jump zone!!!
2562	*/
2563	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2564	{
2565	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2566	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2567	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2568	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2569	AssertRCReturn(rc, rc);
2570	return rc;
2571	}
2572
2573
2574	/**
2575	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2576	* in the VMCS.
2577	*
2578	* @returns VBox status code.
2579	* @param pVM Pointer to the VM.
2580	* @param pVCpu Pointer to the VMCPU.
2581	* @param pCtx Pointer to the guest-CPU context.
2582	*
2583	* @remarks No-long-jump zone!!!
2584	*/
2585	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2586	{
2587	int rc = VINF_SUCCESS;
2588
2589	/*
2590	* Guest CR0.
2591	* Guest FPU.
2592	*/
2593	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2594	{
2595	uint64_t u64GuestCR0 = pCtx->cr0;
2596
2597	/* The guest's view (read access) of its CR0 is unblemished. */
2598	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2599	AssertRCReturn(rc, rc);
2600	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2601
2602	/* Setup VT-x's view of the guest CR0. */
2603	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2604	if (pVM->hm.s.fNestedPaging)
2605	{
2606	if (CPUMIsGuestPagingEnabledEx(pCtx))
2607	{
2608	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2609	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2610	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2611	}
2612	else
2613	{
2614	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2615	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2616	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2617	}
2618
2619	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2620	AssertRCReturn(rc, rc);
2621	}
2622	else
2623	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2624
2625	/*
2626	* Guest FPU bits.
2627	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2628	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2629	*/
2630	u64GuestCR0 \|= X86_CR0_NE;
2631	bool fInterceptNM = false;
2632	if (CPUMIsGuestFPUStateActive(pVCpu))
2633	{
2634	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2635	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2636	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2637	}
2638	else
2639	{
2640	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2641	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2642	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2643	}
2644
2645	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2646	bool fInterceptMF = false;
2647	if (!(pCtx->cr0 & X86_CR0_NE))
2648	fInterceptMF = true;
2649
2650	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2651	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2652	{
2653	Assert(PDMVmmDevHeapIsEnabled(pVM));
2654	Assert(pVM->hm.s.vmx.pRealModeTSS);
2655	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2656	fInterceptNM = true;
2657	fInterceptMF = true;
2658	}
2659	else
2660	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2661
2662	if (fInterceptNM)
2663	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2664	else
2665	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2666
2667	if (fInterceptMF)
2668	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2669	else
2670	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2671
2672	/* Additional intercepts for debugging, define these yourself explicitly. */
2673	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2674	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2675	\| RT_BIT(X86_XCPT_DB)
2676	\| RT_BIT(X86_XCPT_DE)
2677	\| RT_BIT(X86_XCPT_NM)
2678	\| RT_BIT(X86_XCPT_UD)
2679	\| RT_BIT(X86_XCPT_NP)
2680	\| RT_BIT(X86_XCPT_SS)
2681	\| RT_BIT(X86_XCPT_GP)
2682	\| RT_BIT(X86_XCPT_PF)
2683	\| RT_BIT(X86_XCPT_MF);
2684	#elif defined(VBOX_ALWAYS_TRAP_PF)
2685	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2686	#endif
2687
2688	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2689
2690	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2691	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2692	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2693	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2694	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2695	else
2696	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2697
2698	u64GuestCR0 \|= uSetCR0;
2699	u64GuestCR0 &= uZapCR0;
2700	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2701
2702	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2703	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2704	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2705	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2706
2707	/*
2708	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2709	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2710	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2711	*/
2712	uint64_t u64CR0Mask = 0;
2713	u64CR0Mask = X86_CR0_PE
2714	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2715	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2716	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2717	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2718	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2719
2720	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2721	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2722	u64CR0Mask &= ~X86_CR0_PE;
2723
2724	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2725	if (fInterceptNM)
2726	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2727	else
2728	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2729
2730	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2731	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2732	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2733	AssertRCReturn(rc, rc);
2734
2735	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2736	}
2737
2738	/*
2739	* Guest CR2.
2740	* It's always loaded in the assembler code. Nothing to do here.
2741	*/
2742
2743	/*
2744	* Guest CR3.
2745	*/
2746	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2747	{
2748	uint64_t u64GuestCR3 = 0;
2749	if (pVM->hm.s.fNestedPaging)
2750	{
2751	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2752
2753	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2754	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2755	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2756	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2757
2758	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2759	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2760	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2761
2762	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2763	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2764	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2765	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2766
2767	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2768	AssertRCReturn(rc, rc);
2769	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2770
2771	if ( CPUMIsGuestPagingEnabledEx(pCtx)
2772	\|\| pVM->hm.s.vmx.fUnrestrictedGuest)
2773	{
2774	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2775	if (CPUMIsGuestInPAEModeEx(pCtx))
2776	{
2777	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2778	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2779	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2780	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2781	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2782	AssertRCReturn(rc, rc);
2783	}
2784
2785	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2786	have Unrestricted Execution to handle the guest when it's not using paging. */
2787	u64GuestCR3 = pCtx->cr3;
2788	}
2789	else
2790	{
2791	/*
2792	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2793	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2794	* EPT takes care of translating it to host-physical addresses.
2795	*/
2796	RTGCPHYS GCPhys;
2797	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2798	Assert(PDMVmmDevHeapIsEnabled(pVM));
2799
2800	/* We obtain it here every time as the guest could have relocated this PCI region. */
2801	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2802	AssertRCReturn(rc, rc);
2803
2804	u64GuestCR3 = GCPhys;
2805	}
2806	}
2807	else
2808	{
2809	/* Non-nested paging case, just use the hypervisor's CR3. */
2810	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2811	}
2812
2813	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2814	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2815	AssertRCReturn(rc, rc);
2816
2817	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2818	}
2819
2820	/*
2821	* Guest CR4.
2822	*/
2823	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2824	{
2825	uint64_t u64GuestCR4 = pCtx->cr4;
2826
2827	/* The guest's view of its CR4 is unblemished. */
2828	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2829	AssertRCReturn(rc, rc);
2830	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2831
2832	/* Setup VT-x's view of the guest CR4. */
2833	/*
2834	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2835	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2836	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2837	*/
2838	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2839	{
2840	Assert(pVM->hm.s.vmx.pRealModeTSS);
2841	Assert(PDMVmmDevHeapIsEnabled(pVM));
2842	u64GuestCR4 &= ~X86_CR4_VME;
2843	}
2844
2845	if (pVM->hm.s.fNestedPaging)
2846	{
2847	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2848	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2849	{
2850	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2851	u64GuestCR4 \|= X86_CR4_PSE;
2852	/* Our identity mapping is a 32 bits page directory. */
2853	u64GuestCR4 &= ~X86_CR4_PAE;
2854	}
2855	/* else use guest CR4.*/
2856	}
2857	else
2858	{
2859	/*
2860	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2861	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2862	*/
2863	switch (pVCpu->hm.s.enmShadowMode)
2864	{
2865	case PGMMODE_REAL: /* Real-mode. */
2866	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2867	case PGMMODE_32_BIT: /* 32-bit paging. */
2868	{
2869	u64GuestCR4 &= ~X86_CR4_PAE;
2870	break;
2871	}
2872
2873	case PGMMODE_PAE: /* PAE paging. */
2874	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2875	{
2876	u64GuestCR4 \|= X86_CR4_PAE;
2877	break;
2878	}
2879
2880	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2881	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2882	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2883	break;
2884	#endif
2885	default:
2886	AssertFailed();
2887	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2888	}
2889	}
2890
2891	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2892	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2893	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2894	u64GuestCR4 \|= uSetCR4;
2895	u64GuestCR4 &= uZapCR4;
2896
2897	/* Write VT-x's view of the guest CR4 into the VMCS. */
2898	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2899	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2900
2901	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2902	uint64_t u64CR4Mask = 0;
2903	u64CR4Mask = X86_CR4_VME
2904	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2905	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2906	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2907	\| X86_CR4_VMXE;
2908	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2909	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2910	AssertRCReturn(rc, rc);
2911
2912	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2913	}
2914	return rc;
2915	}
2916
2917
2918	/**
2919	* Loads the guest debug registers into the guest-state area in the VMCS.
2920	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2921	*
2922	* @returns VBox status code.
2923	* @param pVM Pointer to the VM.
2924	* @param pVCpu Pointer to the VMCPU.
2925	* @param pCtx Pointer to the guest-CPU context.
2926	*
2927	* @remarks No-long-jump zone!!!
2928	*/
2929	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2930	{
2931	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2932	return VINF_SUCCESS;
2933
2934	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2935	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2936	{
2937	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2938	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2939	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2940	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2941	}
2942
2943	int rc = VERR_INTERNAL_ERROR_5;
2944	bool fInterceptDB = false;
2945	bool fInterceptMovDRx = false;
2946	if (DBGFIsStepping(pVCpu))
2947	{
2948	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2949	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2950	{
2951	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2952	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2953	AssertRCReturn(rc, rc);
2954	Assert(fInterceptDB == false);
2955	}
2956	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2957	{
2958	Assert(fInterceptDB == false);
2959	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2960	* just continuing? */
2961	}
2962	else
2963	fInterceptDB = true;
2964	}
2965	else
2966	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2967
2968	/*
2969	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2970	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2971	* The same for the hypervisor DRx registers, priority is for the guest here.
2972	*/
2973	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
2974	&& !CPUMIsGuestDebugStateActive(pVCpu))
2975	{
2976	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
2977	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2978	AssertRCReturn(rc, rc);
2979	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2980	Assert(fInterceptMovDRx == false);
2981	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2982	}
2983	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
2984	&& !CPUMIsHyperDebugStateActive(pVCpu))
2985	{
2986	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
2987	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2988	AssertRCReturn(rc, rc);
2989	Assert(CPUMIsHyperDebugStateActive(pVCpu));
2990	fInterceptMovDRx = true;
2991	}
2992	else
2993	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
2994
2995	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
2996	if (fInterceptDB)
2997	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
2998	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2999	{
3000	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3001	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3002	#endif
3003	}
3004
3005	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3006	if (fInterceptMovDRx)
3007	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3008	else
3009	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3010
3011	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3012	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3013
3014	/* The guest's view of its DR7 is unblemished. */
3015	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3016
3017	/* Setup other debug controls */
3018	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
3019	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
3020	AssertRCReturn(rc, rc);
3021
3022	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3023	return rc;
3024	}
3025
3026
3027	#ifdef DEBUG
3028	/**
3029	* Debug function to validate segment registers.
3030	*/
3031	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3032	{
3033	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3034	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3035	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3036	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3037	{
3038	/** @todo DPL checks for CS, SS. */
3039	/* Protected mode checks */
3040	/* CS */
3041	Assert(pCtx->cs.Attr.n.u1Present);
3042	Assert(!(pCtx->cs.Attr.u & 0xf00));
3043	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3044	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3045	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3046	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3047	\|\| (pCtx->cs.Attr.n.u1Granularity));
3048	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3049	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3050	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3051	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3052	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3053	else
3054	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3055	/* SS */
3056	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3057	{
3058	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3059	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3060	Assert(pCtx->ss.Attr.n.u1Present);
3061	Assert(!(pCtx->ss.Attr.u & 0xf00));
3062	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3063	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3064	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3065	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3066	\|\| (pCtx->ss.Attr.n.u1Granularity));
3067	}
3068	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3069	/* CR0 might not be up-to-date here always, hence disabled. */
3070	#if 0
3071	if (!pCtx->cr0 & X86_CR0_PE)
3072	Assert(!pCtx->ss.Attr.n.u2Dpl);
3073	#endif
3074	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3075	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3076	{
3077	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3078	Assert(pCtx->ds.Attr.n.u1Present);
3079	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3080	Assert(!(pCtx->ds.Attr.u & 0xf00));
3081	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3082	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3083	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3084	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3085	\|\| (pCtx->ds.Attr.n.u1Granularity));
3086	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3087	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3088	}
3089	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3090	{
3091	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3092	Assert(pCtx->es.Attr.n.u1Present);
3093	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3094	Assert(!(pCtx->es.Attr.u & 0xf00));
3095	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3096	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3097	\|\| !(pCtx->es.Attr.n.u1Granularity));
3098	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3099	\|\| (pCtx->es.Attr.n.u1Granularity));
3100	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3101	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3102	}
3103	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3104	{
3105	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3106	Assert(pCtx->fs.Attr.n.u1Present);
3107	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3108	Assert(!(pCtx->fs.Attr.u & 0xf00));
3109	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3110	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3111	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3112	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3113	\|\| (pCtx->fs.Attr.n.u1Granularity));
3114	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3115	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3116	}
3117	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3118	{
3119	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3120	Assert(pCtx->gs.Attr.n.u1Present);
3121	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3122	Assert(!(pCtx->gs.Attr.u & 0xf00));
3123	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3124	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3125	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3126	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3127	\|\| (pCtx->gs.Attr.n.u1Granularity));
3128	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3129	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3130	}
3131	/* 64-bit capable CPUs. */
3132	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3133	Assert(!(pCtx->cs.u64Base >> 32));
3134	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3135	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3136	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3137	# endif
3138	}
3139	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3140	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3141	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3142	{
3143	/* Real and v86 mode checks. */
3144	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3145	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3146	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3147	{
3148	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3149	}
3150	else
3151	{
3152	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3153	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3154	}
3155
3156	/* CS */
3157	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3158	Assert(pCtx->cs.u32Limit == 0xffff);
3159	Assert(u32CSAttr == 0xf3);
3160	/* SS */
3161	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3162	Assert(pCtx->ss.u32Limit == 0xffff);
3163	Assert(u32SSAttr == 0xf3);
3164	/* DS */
3165	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3166	Assert(pCtx->ds.u32Limit == 0xffff);
3167	Assert(u32DSAttr == 0xf3);
3168	/* ES */
3169	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3170	Assert(pCtx->es.u32Limit == 0xffff);
3171	Assert(u32ESAttr == 0xf3);
3172	/* FS */
3173	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3174	Assert(pCtx->fs.u32Limit == 0xffff);
3175	Assert(u32FSAttr == 0xf3);
3176	/* GS */
3177	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3178	Assert(pCtx->gs.u32Limit == 0xffff);
3179	Assert(u32GSAttr == 0xf3);
3180	/* 64-bit capable CPUs. */
3181	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3182	Assert(!(pCtx->cs.u64Base >> 32));
3183	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3184	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3185	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3186	# endif
3187	}
3188	}
3189	#endif /* DEBUG */
3190
3191
3192	/**
3193	* Writes a guest segment register into the guest-state area in the VMCS.
3194	*
3195	* @returns VBox status code.
3196	* @param pVM Pointer to the VM.
3197	* @param pVCpu Pointer to the VMCPU.
3198	* @param idxSel Index of the selector in the VMCS.
3199	* @param idxLimit Index of the segment limit in the VMCS.
3200	* @param idxBase Index of the segment base in the VMCS.
3201	* @param idxAccess Index of the access rights of the segment in the VMCS.
3202	* @param pSelReg Pointer to the segment selector.
3203	* @param pCtx Pointer to the guest-CPU context.
3204	*
3205	* @remarks No-long-jump zone!!!
3206	*/
3207	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3208	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3209	{
3210	int rc;
3211	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3212	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3213	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3214	AssertRCReturn(rc, rc);
3215
3216	uint32_t u32Access = pSelReg->Attr.u;
3217	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3218	{
3219	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3220	u32Access = 0xf3;
3221	Assert(pVM->hm.s.vmx.pRealModeTSS);
3222	Assert(PDMVmmDevHeapIsEnabled(pVM));
3223	}
3224	else
3225	{
3226	/*
3227	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3228	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3229	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3230	* loaded in protected-mode have their attribute as 0.
3231	*/
3232	if (!u32Access)
3233	u32Access = VMX_SEL_UNUSABLE;
3234	}
3235
3236	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3237	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3238	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3239
3240	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3241	AssertRCReturn(rc, rc);
3242	return rc;
3243	}
3244
3245
3246	/**
3247	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3248	* into the guest-state area in the VMCS.
3249	*
3250	* @returns VBox status code.
3251	* @param pVM Pointer to the VM.
3252	* @param pVCPU Pointer to the VMCPU.
3253	* @param pCtx Pointer to the guest-CPU context.
3254	*
3255	* @remarks No-long-jump zone!!!
3256	* @remarks Requires RFLAGS (for debug assertions).
3257	*/
3258	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3259	{
3260	int rc = VERR_INTERNAL_ERROR_5;
3261
3262	/*
3263	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3264	*/
3265	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3266	{
3267	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3268	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3269	{
3270	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3271	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3272	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3273	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3274	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3275	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3276	}
3277
3278	#ifdef VBOX_WITH_REM
3279	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3280	{
3281	Assert(pVM->hm.s.vmx.pRealModeTSS);
3282	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3283	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3284	{
3285	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3286	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3287	&& enmGuestMode >= PGMMODE_PROTECTED)
3288	{
3289	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3290	in real-mode (e.g. OpenBSD 4.0) */
3291	REMFlushTBs(pVM);
3292	Log2(("Switch to protected mode detected!\n"));
3293	}
3294	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3295	}
3296	}
3297	#endif
3298	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3299	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3300	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3301	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3302	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3303	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3304	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3305	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3306	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3307	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3308	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3309	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3310	AssertRCReturn(rc, rc);
3311
3312	#ifdef DEBUG
3313	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3314	#endif
3315	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3316	}
3317
3318	/*
3319	* Guest TR.
3320	*/
3321	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3322	{
3323	/*
3324	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3325	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3326	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3327	*/
3328	uint16_t u16Sel = 0;
3329	uint32_t u32Limit = 0;
3330	uint64_t u64Base = 0;
3331	uint32_t u32AccessRights = 0;
3332
3333	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3334	{
3335	u16Sel = pCtx->tr.Sel;
3336	u32Limit = pCtx->tr.u32Limit;
3337	u64Base = pCtx->tr.u64Base;
3338	u32AccessRights = pCtx->tr.Attr.u;
3339	}
3340	else
3341	{
3342	Assert(pVM->hm.s.vmx.pRealModeTSS);
3343	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3344
3345	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3346	RTGCPHYS GCPhys;
3347	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3348	AssertRCReturn(rc, rc);
3349
3350	X86DESCATTR DescAttr;
3351	DescAttr.u = 0;
3352	DescAttr.n.u1Present = 1;
3353	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3354
3355	u16Sel = 0;
3356	u32Limit = HM_VTX_TSS_SIZE;
3357	u64Base = GCPhys; /* in real-mode phys = virt. */
3358	u32AccessRights = DescAttr.u;
3359	}
3360
3361	/* Validate. */
3362	Assert(!(u16Sel & RT_BIT(2)));
3363	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3364	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3365	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3366	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3367	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3368	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3369	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3370	Assert( (u32Limit & 0xfff) == 0xfff
3371	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3372	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3373	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3374
3375	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3376	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3377	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3378	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3379	AssertRCReturn(rc, rc);
3380
3381	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3382	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3383	}
3384
3385	/*
3386	* Guest GDTR.
3387	*/
3388	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3389	{
3390	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3391	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3392	AssertRCReturn(rc, rc);
3393
3394	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3395	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3396	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3397	}
3398
3399	/*
3400	* Guest LDTR.
3401	*/
3402	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3403	{
3404	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3405	uint32_t u32Access = 0;
3406	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3407	u32Access = VMX_SEL_UNUSABLE;
3408	else
3409	u32Access = pCtx->ldtr.Attr.u;
3410
3411	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3412	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3413	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3414	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3415	AssertRCReturn(rc, rc);
3416
3417	/* Validate. */
3418	if (!(u32Access & VMX_SEL_UNUSABLE))
3419	{
3420	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3421	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3422	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3423	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3424	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3425	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3426	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3427	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3428	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3429	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3430	}
3431
3432	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3433	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3434	}
3435
3436	/*
3437	* Guest IDTR.
3438	*/
3439	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3440	{
3441	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3442	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3443	AssertRCReturn(rc, rc);
3444
3445	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3446	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3447	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3448	}
3449
3450	/*
3451	* Guest FS & GS base MSRs.
3452	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3453	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3454	* are only available in 64-bit mode.
3455	*/
3456	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3457	* should not be necessary to do it in assembly again. */
3458	if (CPUMIsGuestInLongModeEx(pCtx))
3459	{
3460	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3461	{
3462	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3463	AssertRCReturn(rc, rc);
3464	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3465	}
3466
3467	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3468	{
3469	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3470	AssertRCReturn(rc, rc);
3471	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3472	}
3473	}
3474	else
3475	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3476
3477	return VINF_SUCCESS;
3478	}
3479
3480
3481	/**
3482	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3483	* areas. These MSRs will automatically be loaded to the host CPU on every
3484	* successful VM entry and stored from the host CPU on every successful VM exit.
3485	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3486	*
3487	* @returns VBox status code.
3488	* @param pVM Pointer to the VM.
3489	* @param pVCpu Pointer to the VMCPU.
3490	* @param pCtx Pointer to the guest-CPU context.
3491	*
3492	* @remarks No-long-jump zone!!!
3493	*/
3494	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3495	{
3496	AssertPtr(pVCpu);
3497	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3498
3499	/*
3500	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3501	*/
3502	int rc = VINF_SUCCESS;
3503	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3504	{
3505	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3506	unsigned cGuestMsrs = 0;
3507
3508	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3509	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3510	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3511	if (fSupportsNX \|\| fSupportsLongMode)
3512	{
3513	/** @todo support save IA32_EFER, i.e.
3514	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3515	* guest EFER need not be part of the VM-entry MSR-load area. */
3516	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3517	pGuestMsr->u32Reserved = 0;
3518	pGuestMsr->u64Value = pCtx->msrEFER;
3519	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3520	if (!CPUMIsGuestInLongModeEx(pCtx))
3521	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3522	pGuestMsr++; cGuestMsrs++;
3523	if (fSupportsLongMode)
3524	{
3525	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3526	pGuestMsr->u32Reserved = 0;
3527	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3528	pGuestMsr++; cGuestMsrs++;
3529	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3530	pGuestMsr->u32Reserved = 0;
3531	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3532	pGuestMsr++; cGuestMsrs++;
3533	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3534	pGuestMsr->u32Reserved = 0;
3535	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3536	pGuestMsr++; cGuestMsrs++;
3537	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3538	#if 0
3539	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3540	pGuestMsr->u32Reserved = 0;
3541	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3542	pGuestMsr++; cGuestMsrs++;
3543	#endif
3544	}
3545	}
3546
3547	/*
3548	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3549	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3550	*/
3551	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3552	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3553	{
3554	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3555	pGuestMsr->u32Reserved = 0;
3556	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3557	AssertRCReturn(rc, rc);
3558	pGuestMsr++; cGuestMsrs++;
3559	}
3560
3561	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3562	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3563	{
3564	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3565	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3566	}
3567
3568	/* Update the VCPU's copy of the guest MSR count. */
3569	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3570	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3571	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3572	AssertRCReturn(rc, rc);
3573
3574	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3575	}
3576
3577	/*
3578	* Guest Sysenter MSRs.
3579	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3580	* VM exits on WRMSRs for these MSRs.
3581	*/
3582	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3583	{
3584	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3585	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3586	}
3587	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3588	{
3589	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3590	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3591	}
3592	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3593	{
3594	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3595	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3596	}
3597	AssertRCReturn(rc, rc);
3598
3599	return rc;
3600	}
3601
3602
3603	/**
3604	* Loads the guest activity state into the guest-state area in the VMCS.
3605	*
3606	* @returns VBox status code.
3607	* @param pVM Pointer to the VM.
3608	* @param pVCpu Pointer to the VMCPU.
3609	* @param pCtx Pointer to the guest-CPU context.
3610	*
3611	* @remarks No-long-jump zone!!!
3612	*/
3613	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3614	{
3615	/** @todo See if we can make use of other states, e.g.
3616	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3617	int rc = VINF_SUCCESS;
3618	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3619	{
3620	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3621	AssertRCReturn(rc, rc);
3622	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3623	}
3624	return rc;
3625	}
3626
3627
3628	/**
3629	* Loads the guest-interruptibility state (or "interrupt shadow" as AMD calls
3630	* it) into the guest-state area in the VMCS.
3631	*
3632	* @returns VBox status code.
3633	* @param pVM Pointer to the VM.
3634	* @param pVCpu Pointer to the VMCPU.
3635	* @param pCtx Pointer to the guest-CPU context.
3636	*
3637	* @remarks No-long-jump zone!!!
3638	* @remarks Requires RIP, RFLAGS (debug assert).
3639	*/
3640	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3641	{
3642	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_INTR_STATE))
3643	return VINF_SUCCESS;
3644
3645	/*
3646	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3647	* inhibit interrupts or clear any existing interrupt-inhibition.
3648	*/
3649	uint32_t uIntrState = 0;
3650	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3651	{
3652	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3653	{
3654	/*
3655	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in VT-x
3656	* the flag's condition to be cleared is met and thus the cleared state is correct. Additionally, this means
3657	* we need not re-read the VMCS field on the VM-exit path and clear/set this flag on every VM-exit. Finally,
3658	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
3659	*/
3660	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3661	uIntrState = 0; /* Clear interrupt inhibition. */
3662	}
3663	else if (pCtx->eflags.u32 & X86_EFL_IF)
3664	{
3665	/*
3666	* We don't have enough information to distinguish a block-by-STI vs. block-by-MOV SS. Intel seems to think there
3667	* is a slight difference regarding MOV SS additionally blocking some debug exceptions.
3668	* See Intel spec. 24.2.2 "Guest Non-Register State" table "Format of Interruptibility State".
3669	*/
3670	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3671	}
3672	}
3673	else
3674	uIntrState = 0; /* No interrupt inhibition. */
3675
3676	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3677	Assert((pCtx->eflags.u32 & X86_EFL_IF) \|\| uIntrState == 0); /* If EFLAGS.IF is not set, no interrupt inhibition. */
3678	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3679	AssertRCReturn(rc ,rc);
3680	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_INTR_STATE;
3681	return rc;
3682	}
3683
3684
3685	/**
3686	* Sets up the appropriate function to run guest code.
3687	*
3688	* @returns VBox status code.
3689	* @param pVM Pointer to the VM.
3690	* @param pVCpu Pointer to the VMCPU.
3691	* @param pCtx Pointer to the guest-CPU context.
3692	*
3693	* @remarks No-long-jump zone!!!
3694	* @todo change this to return void.
3695	*/
3696	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3697	{
3698	if (CPUMIsGuestInLongModeEx(pCtx))
3699	{
3700	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3701	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3702	#endif
3703	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3704	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3705	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3706	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3707	#else
3708	/* 64-bit host or hybrid host. */
3709	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3710	#endif
3711	}
3712	else
3713	{
3714	/* Guest is not in long mode, use the 32-bit handler. */
3715	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3716	}
3717	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3718	return VINF_SUCCESS;
3719	}
3720
3721
3722	/**
3723	* Wrapper for running the guest code in VT-x.
3724	*
3725	* @returns VBox strict status code.
3726	* @param pVM Pointer to the VM.
3727	* @param pVCpu Pointer to the VMCPU.
3728	* @param pCtx Pointer to the guest-CPU context.
3729	*
3730	* @remarks No-long-jump zone!!!
3731	*/
3732	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3733	{
3734	/*
3735	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3736	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3737	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3738	*/
3739	#ifdef VBOX_WITH_KERNEL_USING_XMM
3740	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3741	#else
3742	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3743	#endif
3744	}
3745
3746
3747	/**
3748	* Report world-switch error and dump some useful debug info.
3749	*
3750	* @param pVM Pointer to the VM.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3753	* @param pCtx Pointer to the guest-CPU context.
3754	* @param pVmxTransient Pointer to the VMX transient structure (only
3755	* exitReason updated).
3756	*/
3757	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3758	{
3759	Assert(pVM);
3760	Assert(pVCpu);
3761	Assert(pCtx);
3762	Assert(pVmxTransient);
3763	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3764
3765	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3766	switch (rcVMRun)
3767	{
3768	case VERR_VMX_INVALID_VMXON_PTR:
3769	AssertFailed();
3770	break;
3771	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3772	case VERR_VMX_UNABLE_TO_START_VM:
3773	case VERR_VMX_UNABLE_TO_RESUME_VM:
3774	{
3775	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3776	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3777	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3778	AssertRC(rc);
3779
3780	#ifdef VBOX_STRICT
3781	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3782	pVmxTransient->uExitReason));
3783	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3784	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3785	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3786	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3787	else
3788	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3789
3790	/* VMX control bits. */
3791	uint32_t u32Val;
3792	uint64_t u64Val;
3793	RTHCUINTREG uHCReg;
3794	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3795	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3796	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3797	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3798	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3799	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3800	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3801	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3802	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3803	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3804	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3805	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3806	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3807	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3808	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3809	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3810	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3811	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3812	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3813	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3814	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3815	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3816	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3817	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3818	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3819	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3820	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3821	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3822	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3823	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3824	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3826	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3827	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3828	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3829	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3830	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3831	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3832	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3833	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3834	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3835	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3836
3837	/* Guest bits. */
3838	RTGCUINTREG uGCReg;
3839	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3840	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3841	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3842	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3843	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3844	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3845	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3846	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3847
3848	/* Host bits. */
3849	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3850	Log(("Host CR0 %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3852	Log(("Host CR3 %#RHr\n", uHCReg));
3853	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3854	Log(("Host CR4 %#RHr\n", uHCReg));
3855
3856	RTGDTR HostGdtr;
3857	PCX86DESCHC pDesc;
3858	ASMGetGDTR(&HostGdtr);
3859	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3860	Log(("Host CS %#08x\n", u32Val));
3861	if (u32Val < HostGdtr.cbGdt)
3862	{
3863	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3864	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3865	}
3866
3867	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3868	Log(("Host DS %#08x\n", u32Val));
3869	if (u32Val < HostGdtr.cbGdt)
3870	{
3871	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3872	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3873	}
3874
3875	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3876	Log(("Host ES %#08x\n", u32Val));
3877	if (u32Val < HostGdtr.cbGdt)
3878	{
3879	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3880	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3881	}
3882
3883	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3884	Log(("Host FS %#08x\n", u32Val));
3885	if (u32Val < HostGdtr.cbGdt)
3886	{
3887	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3888	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3889	}
3890
3891	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3892	Log(("Host GS %#08x\n", u32Val));
3893	if (u32Val < HostGdtr.cbGdt)
3894	{
3895	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3896	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3897	}
3898
3899	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3900	Log(("Host SS %#08x\n", u32Val));
3901	if (u32Val < HostGdtr.cbGdt)
3902	{
3903	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3904	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3905	}
3906
3907	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3908	Log(("Host TR %#08x\n", u32Val));
3909	if (u32Val < HostGdtr.cbGdt)
3910	{
3911	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3912	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3913	}
3914
3915	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3916	Log(("Host TR Base %#RHv\n", uHCReg));
3917	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3918	Log(("Host GDTR Base %#RHv\n", uHCReg));
3919	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3920	Log(("Host IDTR Base %#RHv\n", uHCReg));
3921	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3922	Log(("Host SYSENTER CS %#08x\n", u32Val));
3923	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3924	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3925	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3926	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3927	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3928	Log(("Host RSP %#RHv\n", uHCReg));
3929	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3930	Log(("Host RIP %#RHv\n", uHCReg));
3931	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3932	if (VMX_IS_64BIT_HOST_MODE())
3933	{
3934	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3935	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3936	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3937	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3938	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3939	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3940	}
3941	# endif
3942	#endif /* VBOX_STRICT */
3943	break;
3944	}
3945
3946	default:
3947	/* Impossible */
3948	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3949	break;
3950	}
3951	NOREF(pVM);
3952	}
3953
3954
3955	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3956	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3957	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3958	#endif
3959
3960	/**
3961	* Executes the specified handler in 64-bit mode.
3962	*
3963	* @returns VBox status code.
3964	* @param pVM Pointer to the VM.
3965	* @param pVCpu Pointer to the VMCPU.
3966	* @param pCtx Pointer to the guest CPU context.
3967	* @param pfnHandler Pointer to the RC handler function.
3968	* @param cbParam Number of parameters.
3969	* @param paParam Array of 32-bit parameters.
3970	*/
3971	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3972	uint32_t *paParam)
3973	{
3974	int rc, rc2;
3975	PHMGLOBLCPUINFO pCpu;
3976	RTHCPHYS HCPhysCpuPage;
3977	RTHCUINTREG uOldEFlags;
3978
3979	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3980	Assert(pfnHandler);
3981	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3982	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3983
3984	#ifdef VBOX_STRICT
3985	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3986	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3987
3988	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3989	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3990	#endif
3991
3992	/* Disable interrupts. */
3993	uOldEFlags = ASMIntDisableFlags();
3994
3995	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3996	RTCPUID idHostCpu = RTMpCpuId();
3997	CPUMR0SetLApic(pVM, idHostCpu);
3998	#endif
3999
4000	pCpu = HMR0GetCurrentCpu();
4001	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4002
4003	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4004	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4005
4006	/* Leave VMX Root Mode. */
4007	VMXDisable();
4008
4009	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4010
4011	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4012	CPUMSetHyperEIP(pVCpu, pfnHandler);
4013	for (int i = (int)cbParam - 1; i >= 0; i--)
4014	CPUMPushHyper(pVCpu, paParam[i]);
4015
4016	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4017
4018	/* Call the switcher. */
4019	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4020	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4021
4022	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4023	/* Make sure the VMX instructions don't cause #UD faults. */
4024	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4025
4026	/* Re-enter VMX Root Mode */
4027	rc2 = VMXEnable(HCPhysCpuPage);
4028	if (RT_FAILURE(rc2))
4029	{
4030	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4031	ASMSetFlags(uOldEFlags);
4032	return VERR_VMX_VMXON_FAILED;
4033	}
4034
4035	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4036	AssertRC(rc2);
4037	Assert(!(ASMGetFlags() & X86_EFL_IF));
4038	ASMSetFlags(uOldEFlags);
4039	return rc;
4040	}
4041
4042
4043	/**
4044	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4045	* supporting 64-bit guests.
4046	*
4047	* @returns VBox status code.
4048	* @param fResume Whether to VMLAUNCH or VMRESUME.
4049	* @param pCtx Pointer to the guest-CPU context.
4050	* @param pCache Pointer to the VMCS cache.
4051	* @param pVM Pointer to the VM.
4052	* @param pVCpu Pointer to the VMCPU.
4053	*/
4054	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4055	{
4056	uint32_t aParam[6];
4057	PHMGLOBLCPUINFO pCpu = NULL;
4058	RTHCPHYS HCPhysCpuPage = 0;
4059	int rc = VERR_INTERNAL_ERROR_5;
4060
4061	pCpu = HMR0GetCurrentCpu();
4062	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4063
4064	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4065	pCache->uPos = 1;
4066	pCache->interPD = PGMGetInterPaeCR3(pVM);
4067	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4068	#endif
4069
4070	#ifdef DEBUG
4071	pCache->TestIn.HCPhysCpuPage= 0;
4072	pCache->TestIn.HCPhysVmcs = 0;
4073	pCache->TestIn.pCache = 0;
4074	pCache->TestOut.HCPhysVmcs = 0;
4075	pCache->TestOut.pCache = 0;
4076	pCache->TestOut.pCtx = 0;
4077	pCache->TestOut.eflags = 0;
4078	#endif
4079
4080	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4081	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4082	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4083	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4084	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4085	aParam[5] = 0;
4086
4087	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4088	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4089	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4090	#endif
4091	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4092
4093	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4094	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4095	Assert(pCtx->dr[4] == 10);
4096	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4097	#endif
4098
4099	#ifdef DEBUG
4100	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4101	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4102	pVCpu->hm.s.vmx.HCPhysVmcs));
4103	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4104	pCache->TestOut.HCPhysVmcs));
4105	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4106	pCache->TestOut.pCache));
4107	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4108	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4109	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4110	pCache->TestOut.pCtx));
4111	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4112	#endif
4113	return rc;
4114	}
4115
4116
4117	/**
4118	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4119	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4120	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4121	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4122	*
4123	* @returns VBox status code.
4124	* @param pVM Pointer to the VM.
4125	* @param pVCpu Pointer to the VMCPU.
4126	*/
4127	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4128	{
4129	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4130	{ \
4131	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4132	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4133	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4134	}
4135
4136	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4137	{ \
4138	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4142	}
4143
4144	AssertPtr(pVM);
4145	AssertPtr(pVCpu);
4146	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4147
4148	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4149	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4150	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4151	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4152	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4153	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4154	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4155	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4156	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4157
4158	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4159	#if 0
4160	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4161	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4162	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4166	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4167	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4169	#endif
4170
4171	/* 32-bit guest-state fields. */
4172	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4173	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4174	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4175	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4176	/* Unused 32-bit guest-state fields. */
4177	#if 0
4178	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4179	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4180	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4181	#endif
4182
4183	/* Natural width guest-state fields. */
4184	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4185	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4186	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4187	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4188	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4189	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4190	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4191	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4192	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4193	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4194	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4195	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4196	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4197	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4198	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4199	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4200	/* Unused natural width guest-state fields. */
4201	#if 0
4202	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4204	#endif
4205	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4207
4208	if (pVM->hm.s.fNestedPaging)
4209	{
4210	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4212	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4213	}
4214	else
4215	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4216
4217	#undef VMXLOCAL_INIT_VMCS_SELREG
4218	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4219	return VINF_SUCCESS;
4220	}
4221
4222
4223	/**
4224	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4225	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4226	* darwin, running 64-bit guests).
4227	*
4228	* @returns VBox status code.
4229	* @param pVCpu Pointer to the VMCPU.
4230	* @param idxField The VMCS field encoding.
4231	* @param u64Val 16, 32 or 64 bits value.
4232	*/
4233	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4234	{
4235	int rc;
4236	switch (idxField)
4237	{
4238	/*
4239	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4240	*/
4241	/* 64-bit Control fields. */
4242	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4243	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4244	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4245	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4246	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4247	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4248	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4249	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4250	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4251	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4252	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4253	case VMX_VMCS64_CTRL_EPTP_FULL:
4254	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4255	/* 64-bit Read-only data fields. */
4256	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4257	/* 64-bit Guest-state fields. */
4258	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4259	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4260	case VMX_VMCS64_GUEST_PAT_FULL:
4261	case VMX_VMCS64_GUEST_EFER_FULL:
4262	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4263	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4264	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4265	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4266	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4267	/* 64-bit Host-state fields. */
4268	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4269	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4270	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4271	{
4272	rc = VMXWriteVmcs32(idxField, u64Val);
4273	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4274	break;
4275	}
4276
4277	/*
4278	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4279	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4280	*/
4281	/* Natural-width Guest-state fields. */
4282	case VMX_VMCS_GUEST_CR0:
4283	case VMX_VMCS_GUEST_CR3:
4284	case VMX_VMCS_GUEST_CR4:
4285	case VMX_VMCS_GUEST_ES_BASE:
4286	case VMX_VMCS_GUEST_CS_BASE:
4287	case VMX_VMCS_GUEST_SS_BASE:
4288	case VMX_VMCS_GUEST_DS_BASE:
4289	case VMX_VMCS_GUEST_FS_BASE:
4290	case VMX_VMCS_GUEST_GS_BASE:
4291	case VMX_VMCS_GUEST_LDTR_BASE:
4292	case VMX_VMCS_GUEST_TR_BASE:
4293	case VMX_VMCS_GUEST_GDTR_BASE:
4294	case VMX_VMCS_GUEST_IDTR_BASE:
4295	case VMX_VMCS_GUEST_DR7:
4296	case VMX_VMCS_GUEST_RSP:
4297	case VMX_VMCS_GUEST_RIP:
4298	case VMX_VMCS_GUEST_RFLAGS:
4299	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4300	case VMX_VMCS_GUEST_SYSENTER_ESP:
4301	case VMX_VMCS_GUEST_SYSENTER_EIP:
4302	{
4303	if ((u64Val >> 32ULL) == 0)
4304	{
4305	/* If this field is 64-bit, VT-x will zero out the top bits. */
4306	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4307	}
4308	else
4309	{
4310	/* Assert that only the 32->64 switcher case should ever come here. */
4311	Assert(pVM->hm.s.fAllow64BitGuests);
4312	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4313	}
4314	break;
4315	}
4316
4317	default:
4318	{
4319	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4320	rc = VERR_INVALID_PARAMETER;
4321	break;
4322	}
4323	}
4324	AssertRCReturn(rc, rc);
4325	return rc;
4326	}
4327
4328
4329	/**
4330	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4331	* hosts (except darwin) for 64-bit guests.
4332	*
4333	* @param pVCpu Pointer to the VMCPU.
4334	* @param idxField The VMCS field encoding.
4335	* @param u64Val 16, 32 or 64 bits value.
4336	*/
4337	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4338	{
4339	AssertPtr(pVCpu);
4340	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4341
4342	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4343	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4344
4345	/* Make sure there are no duplicates. */
4346	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4347	{
4348	if (pCache->Write.aField[i] == idxField)
4349	{
4350	pCache->Write.aFieldVal[i] = u64Val;
4351	return VINF_SUCCESS;
4352	}
4353	}
4354
4355	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4356	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4357	pCache->Write.cValidEntries++;
4358	return VINF_SUCCESS;
4359	}
4360
4361
4362	/**
4363	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4364	*
4365	* @param pCache Pointer to the VMCS cache.
4366	*/
4367	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4368	{
4369	AssertPtr(pCache);
4370	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4371	{
4372	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4373	AssertRC(rc, rc);
4374	}
4375	pCache->Write.cValidEntries = 0;
4376	}
4377
4378
4379	/**
4380	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4381	*
4382	* @param pCache Pointer to the VMCS cache.
4383	* @remarks No-long-jump zone!!!
4384	*/
4385	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4386	{
4387	AssertPtr(pCache);
4388	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4389	{
4390	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4391	AssertRC(rc);
4392	}
4393	}
4394	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4395
4396
4397	/**
4398	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4399	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4400	* timer.
4401	*
4402	* @returns VBox status code.
4403	* @param pVM Pointer to the VM.
4404	* @param pVCpu Pointer to the VMCPU.
4405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4406	* out-of-sync. Make sure to update the required fields
4407	* before using them.
4408	* @remarks No-long-jump zone!!!
4409	*/
4410	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4411	{
4412	int rc = VERR_INTERNAL_ERROR_5;
4413	bool fOffsettedTsc = false;
4414	if (pVM->hm.s.vmx.fUsePreemptTimer)
4415	{
4416	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4417
4418	/* Make sure the returned values have sane upper and lower boundaries. */
4419	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4420	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4421	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4422	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4423
4424	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4425	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4426	}
4427	else
4428	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4429
4430	if (fOffsettedTsc)
4431	{
4432	uint64_t u64CurTSC = ASMReadTSC();
4433	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4434	{
4435	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4436	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4437
4438	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4440	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4441	}
4442	else
4443	{
4444	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4445	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4446	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4447	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4448	}
4449	}
4450	else
4451	{
4452	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4453	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4454	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4455	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4456	}
4457	}
4458
4459
4460	/**
4461	* Determines if an exception is a benign exception. Benign exceptions
4462	* are ones which cannot cause double-faults.
4463	*
4464	* @returns true if the exception is benign, false otherwise.
4465	* @param uVector The exception vector.
4466	*/
4467	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint8_t uVector)
4468	{
4469	switch (uVector)
4470	{
4471	case X86_XCPT_DB:
4472	case X86_XCPT_NMI:
4473	case X86_XCPT_BP:
4474	case X86_XCPT_OF:
4475	case X86_XCPT_BR:
4476	case X86_XCPT_UD:
4477	case X86_XCPT_NM:
4478	case X86_XCPT_CO_SEG_OVERRUN:
4479	case X86_XCPT_MF:
4480	case X86_XCPT_AC:
4481	case X86_XCPT_MC:
4482	case X86_XCPT_XF:
4483	return true;
4484	default:
4485	return false;
4486	}
4487	}
4488
4489
4490	/**
4491	* Determines if an exception is a contributory exception. Contributory
4492	* exceptions are ones which can cause double-faults.
4493	*
4494	* @returns true if the exception is contributory, false otherwise.
4495	* @param uVector The exception vector.
4496	*/
4497	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint8_t uVector)
4498	{
4499	switch (uVector)
4500	{
4501	case X86_XCPT_GP:
4502	case X86_XCPT_SS:
4503	case X86_XCPT_NP:
4504	case X86_XCPT_TS:
4505	case X86_XCPT_DE:
4506	return true;
4507	default:
4508	return false;
4509	}
4510	return false;
4511	}
4512
4513
4514	/**
4515	* Determines if we are intercepting any contributory exceptions.
4516	*
4517	* @returns true if we are intercepting any contributory exception, false
4518	* otherwise.
4519	* @param pVCpu Pointer to the VMCPU.
4520	*/
4521	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4522	{
4523	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4524	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4525	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4526	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4527	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4528	{
4529	return true;
4530	}
4531	return false;
4532	}
4533
4534
4535	/**
4536	* Saves, if necessary, any event that occurred during event delivery as a
4537	* pending VMX event to handle before the next VM-entry or to be translated as a
4538	* TRPM event in the case of exiting to ring-3.
4539	*
4540	* @returns VBox status code (informational error codes included).
4541	* @retval VINF_SUCCESS if we should continue handling VM-exits.
4542	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4543	* continue execution of the guest which will delivery the #DF.
4544	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4545	*
4546	* @param pVM Pointer to the VM.
4547	* @param pVCpu Pointer to the VMCPU.
4548	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4549	* out-of-sync. Make sure to update the required fields
4550	* before using them.
4551	* @param pVmxTransient Pointer to the VMX transient structure.
4552	*
4553	* @remarks No-long-jump zone!!!
4554	* @remarks Called unconditionally after every VM-exit.
4555	*
4556	*/
4557	static int hmR0VmxSavePendingEventDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4558	{
4559
4560	Assert(pVCpu);
4561	Assert(pVmxTransient);
4562
4563	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed)) /* Don't bother with pending events if the VM-entry itself failed. */
4564	return VINF_SUCCESS;
4565
4566	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4567	AssertRCReturn(rc, rc);
4568	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4569	{
4570	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4571	AssertRCReturn(rc, rc);
4572
4573	uint8_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4574	uint8_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4575	uint8_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4576
4577	typedef enum
4578	{
4579	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4580	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4581	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4582	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4583	} VMXREFLECTXCPT;
4584
4585	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4586	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4587	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4588	{
4589	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4590	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4591	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4592	&& uExitVector == X86_XCPT_PF))
4593	{
4594	enmReflect = VMXREFLECTXCPT_XCPT;
4595	}
4596	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4597	&& hmR0VmxIsContributoryXcpt(uExitVector))
4598	{
4599	enmReflect = VMXREFLECTXCPT_DF;
4600	}
4601	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4602	&& uIdtVector == X86_XCPT_PF
4603	&& uExitVector == X86_XCPT_PF)
4604	{
4605	pVmxTransient->fVectoringPF = true;
4606	}
4607	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4608	&& uIdtVector == X86_XCPT_PF
4609	&& hmR0VmxIsContributoryXcpt(uExitVector))
4610	{
4611	enmReflect = VMXREFLECTXCPT_DF;
4612	}
4613	else if (uIdtVector == X86_XCPT_DF)
4614	enmReflect = VMXREFLECTXCPT_TF;
4615	}
4616	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4617	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4618	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4619	{
4620	/*
4621	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4622	* (whatever they are) as they reoccur when restarting the instruction.
4623	*/
4624	enmReflect = VMXREFLECTXCPT_XCPT;
4625	}
4626
4627	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4628	switch (enmReflect)
4629	{
4630	case VMXREFLECTXCPT_XCPT:
4631	{
4632	Assert(!pVCpu->hm.s.Event.fPending);
4633	pVCpu->hm.s.Event.fPending = true;
4634	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4635	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4636	{
4637	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4638	AssertRCReturn(rc, rc);
4639	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4640	}
4641	else
4642	pVCpu->hm.s.Event.u32ErrCode = 0;
4643	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4644	break;
4645	}
4646
4647	case VMXREFLECTXCPT_DF:
4648	{
4649	uint32_t u32IntrInfo;
4650	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4651	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4652	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4653
4654	Assert(!pVCpu->hm.s.Event.fPending);
4655	pVCpu->hm.s.Event.fPending = true;
4656	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4657	pVCpu->hm.s.Event.u32ErrCode = 0;
4658	rc = VINF_VMX_DOUBLE_FAULT;
4659	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4660	break;
4661	}
4662
4663	case VMXREFLECTXCPT_TF:
4664	{
4665	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4666	rc = VINF_EM_RESET;
4667	break;
4668	}
4669
4670	default: /* shut up gcc. */
4671	break;
4672	}
4673	}
4674	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4675	return rc;
4676	}
4677
4678
4679	/**
4680	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4681	*
4682	* @returns VBox status code.
4683	* @param pVM Pointer to the VM.
4684	* @param pVCpu Pointer to the VMCPU.
4685	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4686	* out-of-sync. Make sure to update the required fields
4687	* before using them.
4688	*
4689	* @remarks No-long-jump zone!!!
4690	*/
4691	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4692	{
4693	int rc = VINF_SUCCESS;
4694	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4695	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4696	{
4697	RTGCUINTREG uVal = 0;
4698	RTCCUINTREG uShadow = 0;
4699	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4700	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4701	AssertRCReturn(rc, rc);
4702	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4703	CPUMSetGuestCR0(pVCpu, uVal);
4704	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4705	}
4706	return rc;
4707	}
4708
4709
4710	/**
4711	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4712	*
4713	* @returns VBox status code.
4714	* @param pVM Pointer to the VM.
4715	* @param pVCpu Pointer to the VMCPU.
4716	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4717	* out-of-sync. Make sure to update the required fields
4718	* before using them.
4719	*
4720	* @remarks No-long-jump zone!!!
4721	*/
4722	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4723	{
4724	int rc = VINF_SUCCESS;
4725	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4726	{
4727	RTGCUINTREG uVal = 0;
4728	RTCCUINTREG uShadow = 0;
4729	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4730	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4731	AssertRCReturn(rc, rc);
4732	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4733	CPUMSetGuestCR4(pVCpu, uVal);
4734	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4735	}
4736	return rc;
4737	}
4738
4739
4740	/**
4741	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4742	*
4743	* @returns VBox status code.
4744	* @param pVM Pointer to the VM.
4745	* @param pVCpu Pointer to the VMCPU.
4746	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4747	* out-of-sync. Make sure to update the required fields
4748	* before using them.
4749	*
4750	* @remarks No-long-jump zone!!!
4751	*/
4752	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4753	{
4754	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4755	return VINF_SUCCESS;
4756
4757	RTGCUINTREG uVal = 0;
4758	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4759	AssertRCReturn(rc, rc);
4760	pMixedCtx->rip = uVal;
4761	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4762	return rc;
4763	}
4764
4765
4766	/**
4767	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4768	*
4769	* @returns VBox status code.
4770	* @param pVM Pointer to the VM.
4771	* @param pVCpu Pointer to the VMCPU.
4772	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4773	* out-of-sync. Make sure to update the required fields
4774	* before using them.
4775	*
4776	* @remarks No-long-jump zone!!!
4777	*/
4778	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4779	{
4780	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4781	return VINF_SUCCESS;
4782
4783	RTGCUINTREG uVal = 0;
4784	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4785	AssertRCReturn(rc, rc);
4786	pMixedCtx->rsp = uVal;
4787	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4788	return rc;
4789	}
4790
4791
4792	/**
4793	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4794	*
4795	* @returns VBox status code.
4796	* @param pVM Pointer to the VM.
4797	* @param pVCpu Pointer to the VMCPU.
4798	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4799	* out-of-sync. Make sure to update the required fields
4800	* before using them.
4801	*
4802	* @remarks No-long-jump zone!!!
4803	*/
4804	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4805	{
4806	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4807	return VINF_SUCCESS;
4808
4809	RTGCUINTREG uVal = 0;
4810	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4811	AssertRCReturn(rc, rc);
4812	pMixedCtx->rflags.u64 = uVal;
4813
4814	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4815	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4816	{
4817	Assert(pVM->hm.s.vmx.pRealModeTSS);
4818	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4819	pMixedCtx->eflags.Bits.u1VM = 0;
4820	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4821	}
4822
4823	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4824	return rc;
4825	}
4826
4827
4828	/**
4829	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4830	* guest-CPU context.
4831	*/
4832	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4833	{
4834	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4835	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4836	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4837	AssertRCReturn(rc, rc);
4838	return rc;
4839	}
4840
4841
4842	/**
4843	* Saves the guest's interruptibility state.
4844	*
4845	* @returns VBox status code.
4846	* @param pVM Pointer to the VM.
4847	* @param pVCpu Pointer to the VMCPU.
4848	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4849	* out-of-sync. Make sure to update the required fields
4850	* before using them.
4851	*
4852	* @remarks No-long-jump zone!!!
4853	*/
4854	DECLINLINE(int) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4855	{
4856	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_INTR_STATE)
4857	return VINF_SUCCESS;
4858
4859	uint32_t uIntrState = 0;
4860	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4861	if (uIntrState != 0)
4862	{
4863	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4864	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4865	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4866	AssertRCReturn(rc, rc);
4867	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4868	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4869	}
4870	else
4871	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4872
4873	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_INTR_STATE;
4874	return rc;
4875	}
4876
4877
4878	/**
4879	* Saves the guest's activity state.
4880	*
4881	* @returns VBox status code.
4882	* @param pVM Pointer to the VM.
4883	* @param pVCpu Pointer to the VMCPU.
4884	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4885	* out-of-sync. Make sure to update the required fields
4886	* before using them.
4887	*
4888	* @remarks No-long-jump zone!!!
4889	*/
4890	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4891	{
4892	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4893	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4894	return VINF_SUCCESS;
4895	}
4896
4897
4898	/**
4899	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4900	* the current VMCS into the guest-CPU context.
4901	*
4902	* @returns VBox status code.
4903	* @param pVM Pointer to the VM.
4904	* @param pVCpu Pointer to the VMCPU.
4905	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4906	* out-of-sync. Make sure to update the required fields
4907	* before using them.
4908	*
4909	* @remarks No-long-jump zone!!!
4910	*/
4911	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4912	{
4913	int rc = VINF_SUCCESS;
4914	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4915	{
4916	uint32_t u32Val = 0;
4917	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4918	pMixedCtx->SysEnter.cs = u32Val;
4919	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4920	}
4921
4922	RTGCUINTREG uGCVal = 0;
4923	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4924	{
4925	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4926	pMixedCtx->SysEnter.eip = uGCVal;
4927	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4928	}
4929	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4930	{
4931	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4932	pMixedCtx->SysEnter.esp = uGCVal;
4933	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4934	}
4935	return rc;
4936	}
4937
4938
4939	/**
4940	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4941	* context.
4942	*
4943	* @returns VBox status code.
4944	* @param pVM Pointer to the VM.
4945	* @param pVCpu Pointer to the VMCPU.
4946	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4947	* out-of-sync. Make sure to update the required fields
4948	* before using them.
4949	*
4950	* @remarks No-long-jump zone!!!
4951	*/
4952	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4953	{
4954	RTGCUINTREG uVal = 0;
4955	int rc = VINF_SUCCESS;
4956	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4957	{
4958	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4959	pMixedCtx->fs.u64Base = uVal;
4960	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4961	}
4962	return rc;
4963	}
4964
4965
4966	/**
4967	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4968	* context.
4969	*
4970	* @returns VBox status code.
4971	* @param pVM Pointer to the VM.
4972	* @param pVCpu Pointer to the VMCPU.
4973	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4974	* out-of-sync. Make sure to update the required fields
4975	* before using them.
4976	*
4977	* @remarks No-long-jump zone!!!
4978	*/
4979	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4980	{
4981	RTGCUINTREG uVal = 0;
4982	int rc = VINF_SUCCESS;
4983	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4984	{
4985	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4986	pMixedCtx->gs.u64Base = uVal;
4987	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4988	}
4989	return rc;
4990	}
4991
4992
4993	/**
4994	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4995	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4996	*
4997	* @returns VBox status code.
4998	* @param pVM Pointer to the VM.
4999	* @param pVCpu Pointer to the VMCPU.
5000	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5001	* out-of-sync. Make sure to update the required fields
5002	* before using them.
5003	*
5004	* @remarks No-long-jump zone!!!
5005	*/
5006	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5007	{
5008	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5009	return VINF_SUCCESS;
5010
5011	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5012	{
5013	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5014	pMsr += i;
5015	switch (pMsr->u32IndexMSR)
5016	{
5017	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5018	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5019	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5020	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5021	#if 0
5022	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5023	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5024	#endif
5025	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5026	default:
5027	{
5028	AssertFailed();
5029	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5030	}
5031	}
5032	}
5033	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5034	return VINF_SUCCESS;
5035	}
5036
5037
5038	/**
5039	* Saves the guest control registers from the current VMCS into the guest-CPU
5040	* context.
5041	*
5042	* @returns VBox status code.
5043	* @param pVM Pointer to the VM.
5044	* @param pVCpu Pointer to the VMCPU.
5045	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5046	* out-of-sync. Make sure to update the required fields
5047	* before using them.
5048	*
5049	* @remarks No-long-jump zone!!!
5050	*/
5051	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5052	{
5053	RTGCUINTREG uVal = 0;
5054	RTGCUINTREG uShadow = 0;
5055	int rc = VINF_SUCCESS;
5056
5057	/* Guest CR0. Guest FPU. */
5058	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5059	AssertRCReturn(rc, rc);
5060
5061	/* Guest CR4. */
5062	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5063	AssertRCReturn(rc, rc);
5064
5065	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5066	if ( pVM->hm.s.fNestedPaging
5067	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5068	{
5069	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5070	{
5071	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5072	AssertRCReturn(rc, rc);
5073	if (pMixedCtx->cr3 != uVal)
5074	{
5075	CPUMSetGuestCR3(pVCpu, uVal);
5076	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5077	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5078	}
5079
5080	/* We require EFER to check PAE mode. */
5081	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5082	AssertRCReturn(rc, rc);
5083
5084	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5085	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5086	{
5087	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5088	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5089	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5090	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5091	AssertRCReturn(rc, rc);
5092	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5093	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5094	}
5095
5096	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5097	}
5098	}
5099	else
5100	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5101	return rc;
5102	}
5103
5104
5105	/**
5106	* Reads a guest segment register from the current VMCS into the guest-CPU
5107	* context.
5108	*
5109	* @returns VBox status code.
5110	* @param idxSel Index of the selector in the VMCS.
5111	* @param idxLimit Index of the segment limit in the VMCS.
5112	* @param idxBase Index of the segment base in the VMCS.
5113	* @param idxAccess Index of the access rights of the segment in the VMCS.
5114	* @param pSelReg Pointer to the segment selector.
5115	*
5116	* @remarks No-long-jump zone!!!
5117	*/
5118	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5119	PCPUMSELREG pSelReg)
5120	{
5121	uint32_t u32Val = 0;
5122	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5123	pSelReg->Sel = (uint16_t)u32Val;
5124	pSelReg->ValidSel = (uint16_t)u32Val;
5125	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5126
5127	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5128	pSelReg->u32Limit = u32Val;
5129
5130	RTGCUINTREG uGCVal = 0;
5131	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5132	pSelReg->u64Base = uGCVal;
5133
5134	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5135	pSelReg->Attr.u = u32Val;
5136
5137	/*
5138	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5139	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5140	*/
5141	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5142	{
5143	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5144	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5145	}
5146	return rc;
5147	}
5148
5149
5150	/**
5151	* Saves the guest segment registers from the current VMCS into the guest-CPU
5152	* context.
5153	*
5154	* @returns VBox status code.
5155	* @param pVM Pointer to the VM.
5156	* @param pVCpu Pointer to the VMCPU.
5157	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5158	* out-of-sync. Make sure to update the required fields
5159	* before using them.
5160	*
5161	* @remarks No-long-jump zone!!!
5162	*/
5163	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5164	{
5165	int rc = VINF_SUCCESS;
5166
5167	/* Guest segment registers. */
5168	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5169	{
5170	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5171	AssertRCReturn(rc, rc);
5172
5173	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5174	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5175	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5176	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5177	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5178	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5179	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5180	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5181	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5182	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5183	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5184	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5185	AssertRCReturn(rc, rc);
5186
5187	/* Restore segment attributes for real-on-v86 mode hack. */
5188	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5189	{
5190	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5191	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5192	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5193	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5194	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5195	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5196	}
5197
5198	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5199	}
5200
5201	/* Guest LDTR. */
5202	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5203	{
5204	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5205	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5206	AssertRCReturn(rc, rc);
5207	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5208	}
5209
5210	/* Guest GDTR. */
5211	RTGCUINTREG uGCVal = 0;
5212	uint32_t u32Val = 0;
5213	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5214	{
5215	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5216	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5217	pMixedCtx->gdtr.pGdt = uGCVal;
5218	pMixedCtx->gdtr.cbGdt = u32Val;
5219	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5220	}
5221
5222	/* Guest IDTR. */
5223	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5224	{
5225	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5226	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5227	pMixedCtx->idtr.pIdt = uGCVal;
5228	pMixedCtx->idtr.cbIdt = u32Val;
5229	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5230	}
5231
5232	/* Guest TR. */
5233	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5234	{
5235	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5236	AssertRCReturn(rc, rc);
5237
5238	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5239	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5240	{
5241	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5242	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5243	AssertRCReturn(rc, rc);
5244	}
5245	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5246	}
5247	return rc;
5248	}
5249
5250
5251	/**
5252	* Saves the guest debug registers from the current VMCS into the guest-CPU
5253	* context.
5254	*
5255	* @returns VBox status code.
5256	* @param pVM Pointer to the VM.
5257	* @param pVCpu Pointer to the VMCPU.
5258	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5259	* out-of-sync. Make sure to update the required fields
5260	* before using them.
5261	*
5262	* @remarks No-long-jump zone!!!
5263	*/
5264	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5265	{
5266	int rc = VINF_SUCCESS;
5267	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5268	{
5269	RTGCUINTREG uVal;
5270	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5271	pMixedCtx->dr[7] = uVal;
5272
5273	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5274	}
5275	return rc;
5276	}
5277
5278
5279	/**
5280	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5281	*
5282	* @returns VBox status code.
5283	* @param pVM Pointer to the VM.
5284	* @param pVCpu Pointer to the VMCPU.
5285	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5286	* out-of-sync. Make sure to update the required fields
5287	* before using them.
5288	*
5289	* @remarks No-long-jump zone!!!
5290	*/
5291	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5292	{
5293	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5294	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5295	return VINF_SUCCESS;
5296	}
5297
5298
5299	/**
5300	* Saves the entire guest state from the currently active VMCS into the
5301	* guest-CPU context. This essentially VMREADs all guest-data.
5302	*
5303	* @returns VBox status code.
5304	* @param pVM Pointer to the VM.
5305	* @param pVCpu Pointer to the VMCPU.
5306	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5307	* out-of-sync. Make sure to update the required fields
5308	* before using them.
5309	*/
5310	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5311	{
5312	Assert(pVM);
5313	Assert(pVCpu);
5314	Assert(pMixedCtx);
5315
5316	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5317	return VINF_SUCCESS;
5318
5319	VMMRZCallRing3Disable(pVCpu); /* We're not using hmR0VmxCallRing3Disable() as we want to respect nesting here. */
5320
5321	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5323
5324	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5326
5327	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5329
5330	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5332
5333	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5335
5336	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5338
5339	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5341
5342	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5343	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5344
5345	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
5346	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestIntrState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5347
5348	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5349	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5350
5351	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5352	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5353
5354	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5355	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5356
5357	VMMRZCallRing3Enable(pVCpu);
5358	return rc;
5359	}
5360
5361
5362	/**
5363	* Check per-VM and per-VCPU force flag actions that require us to go back to
5364	* ring-3 for one reason or another.
5365	*
5366	* @returns VBox status code (information status code included).
5367	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5368	* ring-3.
5369	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5370	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5371	* interrupts)
5372	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5373	* all EMTs to be in ring-3.
5374	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5375	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5376	* to the EM loop.
5377	*
5378	* @param pVM Pointer to the VM.
5379	* @param pVCpu Pointer to the VMCPU.
5380	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5381	* out-of-sync. Make sure to update the required fields
5382	* before using them.
5383	*/
5384	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5385	{
5386	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5387
5388	int rc = VERR_INTERNAL_ERROR_5;
5389	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5390	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5391	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5392	{
5393	/* We need the control registers now, make sure the guest-CPU context is updated. */
5394	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5395	AssertRCReturn(rc, rc);
5396
5397	/* Pending HM CR3 sync. */
5398	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5399	{
5400	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5401	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5402	}
5403	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5404	{
5405	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5406	AssertRCReturn(rc, rc);
5407	}
5408
5409	/* Pending PGM C3 sync. */
5410	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5411	{
5412	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5413	if (rc != VINF_SUCCESS)
5414	{
5415	AssertRC(rc);
5416	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5417	return rc;
5418	}
5419	}
5420
5421	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5422	/* -XXX- what was that about single stepping? */
5423	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5424	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5425	{
5426	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5427	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5428	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5429	return rc;
5430	}
5431
5432	/* Pending VM request packets, such as hardware interrupts. */
5433	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5434	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5435	{
5436	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5437	return VINF_EM_PENDING_REQUEST;
5438	}
5439
5440	/* Pending PGM pool flushes. */
5441	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5442	{
5443	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5444	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5445	}
5446
5447	/* Pending DMA requests. */
5448	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5449	{
5450	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5451	return VINF_EM_RAW_TO_R3;
5452	}
5453	}
5454
5455	/* Paranoia. */
5456	Assert(rc != VERR_EM_INTERPRETER);
5457	return VINF_SUCCESS;
5458	}
5459
5460
5461	/**
5462	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5463	* VT-x to execute any instruction.
5464	*
5465	* @param pvCpu Pointer to the VMCPU.
5466	*/
5467	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5468	{
5469	if (pVCpu->hm.s.Event.fPending)
5470	{
5471	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5472	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5473	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5474	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5475
5476	/* If a trap was already pending, we did something wrong! */
5477	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5478
5479	/* A page-fault exception during a page-fault would become a double-fault. */
5480	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5481	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5482
5483	TRPMEVENT enmTrapType;
5484	switch (uVectorType)
5485	{
5486	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5487	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5488	enmTrapType = TRPM_HARDWARE_INT;
5489	break;
5490	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5491	enmTrapType = TRPM_SOFTWARE_INT;
5492	break;
5493	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5494	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5495	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5496	enmTrapType = TRPM_TRAP;
5497	break;
5498	default:
5499	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5500	enmTrapType = TRPM_32BIT_HACK;
5501	break;
5502	}
5503	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5504	AssertRC(rc);
5505	if (fErrorCodeValid)
5506	TRPMSetErrorCode(pVCpu, uErrorCode);
5507
5508	/* Clear the VT-x state bits now that TRPM has the information. */
5509	pVCpu->hm.s.Event.fPending = false;
5510	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5511	AssertRC(rc);
5512	}
5513	}
5514
5515
5516	/**
5517	* Does the necessary state syncing before doing a longjmp to ring-3.
5518	*
5519	* @param pVM Pointer to the VM.
5520	* @param pVCpu Pointer to the VMCPU.
5521	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5522	* out-of-sync. Make sure to update the required fields
5523	* before using them.
5524	* @param rcExit The reason for exiting to ring-3. Can be
5525	* VINF_VMM_UNKNOWN_RING3_CALL.
5526	*
5527	* @remarks No-long-jmp zone!!!
5528	*/
5529	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5530	{
5531	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5532	Log(("hmR0VmxLongJmpToRing3: rcExit=%d\n", rcExit));
5533
5534	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5535	AssertRC(rc);
5536
5537	/* Restore debug registers if necessary and resync on next R0 re-entry. */
5538	if (CPUMIsGuestDebugStateActive(pVCpu))
5539	{
5540	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5541	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5542	}
5543	else if (CPUMIsHyperDebugStateActive(pVCpu))
5544	{
5545	CPUMR0LoadHostDebugState(pVM, pVCpu);
5546	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5547	}
5548
5549	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5550	}
5551
5552
5553	/**
5554	* An action requires us to go back to ring-3. This function does the necessary
5555	* steps before we can safely return to ring-3. This is not the same as longjmps
5556	* to ring-3, this is voluntary.
5557	*
5558	* @param pVM Pointer to the VM.
5559	* @param pVCpu Pointer to the VMCPU.
5560	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5561	* out-of-sync. Make sure to update the required fields
5562	* before using them.
5563	* @param rcExit The reason for exiting to ring-3. Can be
5564	* VINF_VMM_UNKNOWN_RING3_CALL.
5565	*/
5566	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5567	{
5568	Assert(pVM);
5569	Assert(pVCpu);
5570	Assert(pMixedCtx);
5571	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5572
5573	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5574	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5575	return;
5576
5577	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5578	VMMRZCallRing3Disable(pVCpu);
5579	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5580
5581	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5582	hmR0VmxUpdateTRPMTrap(pVCpu);
5583
5584	/* Sync. the guest state. */
5585	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5586	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5587
5588	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5589	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5590	\| CPUM_CHANGED_LDTR
5591	\| CPUM_CHANGED_GDTR
5592	\| CPUM_CHANGED_IDTR
5593	\| CPUM_CHANGED_TR
5594	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5595
5596	/* On our way back from ring-3 the following needs to be done. */
5597	/** @todo This can change with preemption hooks. */
5598	if (rcExit == VINF_EM_RAW_INTERRUPT)
5599	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5600	else
5601	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5602
5603	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5604	VMMRZCallRing3Enable(pVCpu);
5605	}
5606
5607
5608	/**
5609	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5610	* longjump to ring-3 and possibly get preempted.
5611	*
5612	* @param pVCpu Pointer to the VMCPU.
5613	* @param enmOperation The operation causing the ring-3 longjump.
5614	* @param pvUser The user argument (pointer to the possibly
5615	* out-of-date guest-CPU context).
5616	*
5617	* @remarks Must never be called with @a enmOperation ==
5618	* VMMCALLRING3_VM_R0_ASSERTION.
5619	*/
5620	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5621	{
5622	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5623	Assert(pVCpu);
5624	Assert(pvUser);
5625	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5626	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5627
5628	VMMRZCallRing3Disable(pVCpu);
5629	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5630	VMMRZCallRing3Enable(pVCpu);
5631	}
5632
5633
5634	/**
5635	* Injects any pending TRPM trap into the VM by updating the VMCS.
5636	*
5637	* @returns VBox status code (informational status code included).
5638	* @param pVM Pointer to the VM.
5639	* @param pVCpu Pointer to the VMCPU.
5640	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5641	* out-of-sync. Make sure to update the required fields
5642	* before using them.
5643	*/
5644	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5645	{
5646	if (!TRPMHasTrap(pVCpu))
5647	return VINF_SUCCESS;
5648
5649	uint8_t u8Vector = 0;
5650	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5651	RTGCUINT uErrCode = 0;
5652
5653	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5654	AssertRCReturn(rc, rc);
5655	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5656
5657	rc = TRPMResetTrap(pVCpu);
5658	AssertRCReturn(rc, rc);
5659
5660	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5661	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5662	if (enmTrpmEvent == TRPM_TRAP)
5663	{
5664	switch (u8Vector)
5665	{
5666	case X86_XCPT_BP:
5667	case X86_XCPT_OF:
5668	{
5669	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5670	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5671	break;
5672	}
5673
5674	case X86_XCPT_DF:
5675	case X86_XCPT_TS:
5676	case X86_XCPT_NP:
5677	case X86_XCPT_SS:
5678	case X86_XCPT_GP:
5679	case X86_XCPT_PF:
5680	case X86_XCPT_AC:
5681	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5682	them which VT-x/VMM pushes to the guest stack. */
5683	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5684	/* no break! */
5685	default:
5686	{
5687	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5688	break;
5689	}
5690	}
5691	}
5692	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5693	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5694	else
5695	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5696
5697	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5698	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5699	}
5700
5701
5702	/**
5703	* Checks if there are any pending guest interrupts to be delivered and injects
5704	* them into the VM by updating the VMCS.
5705	*
5706	* @returns VBox status code (informational status codes included).
5707	* @param pVM Pointer to the VM.
5708	* @param pVCpu Pointer to the VMCPU.
5709	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5710	* out-of-sync. Make sure to update the required fields
5711	* before using them.
5712	*
5713	* @remarks This must be called only after hmR0VmxSaveGuestIntrState().
5714	*/
5715	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5716	{
5717	/* First inject any pending HM interrupts. */
5718	if (pVCpu->hm.s.Event.fPending)
5719	{
5720	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5721	pVCpu->hm.s.Event.u32ErrCode);
5722	AssertRCReturn(rc, rc);
5723	pVCpu->hm.s.Event.fPending = false;
5724	return rc;
5725	}
5726
5727	/** @todo SMI. SMIs take priority over NMIs. */
5728
5729	/* NMI. NMIs take priority over regular interrupts . */
5730	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5731	{
5732	/* Construct an NMI interrupt and inject it into the VMCS. */
5733	RTGCUINTPTR uIntrInfo;
5734	uIntrInfo = X86_XCPT_NMI;
5735	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5736	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5737	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5738	AssertRCReturn(rc, rc);
5739	return rc;
5740	}
5741
5742	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5743	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5744	AssertRCReturn(rc, rc);
5745
5746	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5747	if (!TRPMHasTrap(pVCpu))
5748	{
5749	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5750	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5751	{
5752	/*
5753	* When external interrupts are pending and the guest has disabled interrupts, cause a VM-exit using "interrupt-window
5754	* exiting" so we can deliver the interrupt when the guest is ready to receive them. Otherwise, if the guest
5755	* can receive interrupts now, convert the PDM interrupt into a TRPM event and inject it.
5756	*/
5757	if (!(pMixedCtx->eflags.u32 & X86_EFL_IF)) /** @todo we can use interrupt-window exiting for block-by-STI. */
5758	{
5759	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5760	{
5761	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5762	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5763	AssertRCReturn(rc, rc);
5764	}
5765	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5766	}
5767	else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5768	{
5769	uint8_t u8Interrupt = 0;
5770	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5771	if (RT_SUCCESS(rc))
5772	{
5773	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5774	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5775	AssertRCReturn(rc, rc);
5776	}
5777	else
5778	{
5779	/** @todo Does this actually happen? If not turn it into an assertion. */
5780	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5781	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5782	}
5783	}
5784	}
5785	}
5786
5787	/* If interrupts can be delivered, inject it into the VM. */
5788	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5789	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5790	&& TRPMHasTrap(pVCpu))
5791	{
5792	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5793	AssertRCReturn(rc, rc);
5794	}
5795	return rc;
5796	}
5797
5798	/**
5799	* Injects an invalid-opcode (#UD) exception into the VM.
5800	*
5801	* @returns VBox status code (informational status code included).
5802	* @param pVM Pointer to the VM.
5803	* @param pVCpu Pointer to the VMCPU.
5804	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5805	* out-of-sync. Make sure to update the required fields
5806	* before using them.
5807	*/
5808	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5809	{
5810	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5811	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5812	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5813	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5814	}
5815
5816
5817	/**
5818	* Injects a double-fault (#DF) exception into the VM.
5819	*
5820	* @returns VBox status code (informational status code included).
5821	* @param pVM Pointer to the VM.
5822	* @param pVCpu Pointer to the VMCPU.
5823	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5824	* out-of-sync. Make sure to update the required fields
5825	* before using them.
5826	*/
5827	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5828	{
5829	/* Inject the double-fault. */
5830	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5831	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5832	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5833	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5834	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5835	}
5836
5837
5838	/**
5839	* Injects a debug (#DB) exception into the VM.
5840	*
5841	* @returns VBox status code (informational status code included).
5842	* @param pVM Pointer to the VM.
5843	* @param pVCpu Pointer to the VMCPU.
5844	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5845	* out-of-sync. Make sure to update the required fields
5846	* before using them.
5847	*/
5848	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5849	{
5850	/* Inject the debug-exception. */
5851	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5852	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5853	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5854	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5855	}
5856
5857
5858	/**
5859	* Injects a overflow (#OF) exception into the VM.
5860	*
5861	* @returns VBox status code (informational status code included).
5862	* @param pVM Pointer to the VM.
5863	* @param pVCpu Pointer to the VMCPU.
5864	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5865	* out-of-sync. Make sure to update the required fields
5866	* before using them.
5867	* @param cbInstr The value of RIP that is to be pushed on the guest
5868	* stack.
5869	*/
5870	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5871	{
5872	/* Inject the overflow exception. */
5873	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5874	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5875	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5876	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5877	}
5878
5879
5880	/**
5881	* Injects a general-protection (#GP) fault into the VM.
5882	*
5883	* @returns VBox status code (informational status code included).
5884	* @param pVM Pointer to the VM.
5885	* @param pVCpu Pointer to the VMCPU.
5886	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5887	* out-of-sync. Make sure to update the required fields
5888	* before using them.
5889	* @param u32ErrorCode The error code associated with the #GP.
5890	*/
5891	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5892	{
5893	/* Inject the general-protection fault. */
5894	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5895	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5896	if (fErrorCodeValid)
5897	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5898	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5899	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5900	}
5901
5902
5903	/**
5904	* Injects a software interrupt (INTn) into the VM.
5905	*
5906	* @returns VBox status code (informational status code included).
5907	* @param pVM Pointer to the VM.
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	* @param uVector The software interrupt vector number.
5913	* @param cbInstr The value of RIP that is to be pushed on the guest
5914	* stack.
5915	*/
5916	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5917	{
5918	/* Inject the INTn. */
5919	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5920	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5921	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5922	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5923	}
5924
5925
5926	/**
5927	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5928	* stack.
5929	*
5930	* @returns VBox status code (information status code included).
5931	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5932	* @param pVM Pointer to the VM.
5933	* @param pMixedCtx Pointer to the guest-CPU context.
5934	* @param uValue The value to push to the guest stack.
5935	*/
5936	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5937	{
5938	/*
5939	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5940	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5941	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5942	*/
5943	if (pMixedCtx->sp == 1)
5944	return VINF_EM_RESET;
5945	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5946	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5947	AssertRCReturn(rc, rc);
5948	return rc;
5949	}
5950
5951
5952	/**
5953	* Injects an event into the guest upon VM-entry by updating the relevant fields
5954	* in the VM-entry area in the VMCS.
5955	*
5956	* @returns VBox status code (informational error codes included).
5957	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5958	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5959	*
5960	* @param pVM Pointer to the VM.
5961	* @param pVCpu Pointer to the VMCPU.
5962	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5963	* out-of-sync. Make sure to update the required fields
5964	* before using them.
5965	* @param u64IntrInfo The VM-entry interruption-information field.
5966	* @param cbInstr The VM-entry instruction length in bytes (for software
5967	* interrupts, exceptions and privileged software
5968	* exceptions).
5969	* @param u32ErrCode The VM-entry exception error code.
5970	*
5971	* @remarks No-long-jump zone!!!
5972	*/
5973	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5974	uint32_t u32ErrCode)
5975	{
5976	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5977	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5978	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5979
5980	/* We require CR0 to check if the guest is in real-mode. */
5981	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5982	AssertRCReturn(rc, rc);
5983
5984	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5985	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5986
5987	/*
5988	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5989	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5990	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5991	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5992	*/
5993	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5994	{
5995	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5996	{
5997	Assert(PDMVmmDevHeapIsEnabled(pVM));
5998	Assert(pVM->hm.s.vmx.pRealModeTSS);
5999
6000	/* Save the required guest state bits from the VMCS. */
6001	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6002	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6003	AssertRCReturn(rc, rc);
6004
6005	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6006	const size_t cbIdtEntry = 4;
6007	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6008	{
6009	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6010	if (uVector == X86_XCPT_DF)
6011	return VINF_EM_RESET;
6012	else if (uVector == X86_XCPT_GP)
6013	{
6014	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6015	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6016	}
6017
6018	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6019	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6020	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6021	}
6022
6023	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6024	uint16_t uGuestIp = pMixedCtx->ip;
6025	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6026	{
6027	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6028	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6029	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6030	}
6031	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6032	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6033
6034	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6035	uint16_t offIdtEntry = 0;
6036	RTSEL selIdtEntry = 0;
6037	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6038	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6039	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6040	AssertRCReturn(rc, rc);
6041
6042	/* Construct the stack frame for the interrupt/exception handler. */
6043	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6044	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6045	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6046	AssertRCReturn(rc, rc);
6047
6048	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6049	if (rc == VINF_SUCCESS)
6050	{
6051	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6052	pMixedCtx->rip = offIdtEntry;
6053	pMixedCtx->cs.Sel = selIdtEntry;
6054	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6055
6056	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6057	\| HM_CHANGED_GUEST_RIP
6058	\| HM_CHANGED_GUEST_RFLAGS
6059	\| HM_CHANGED_GUEST_RSP;
6060	AssertRCReturn(rc, rc);
6061	}
6062	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6063	return rc;
6064	}
6065	else
6066	{
6067	/*
6068	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6069	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6070	*/
6071	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6072	}
6073	}
6074
6075	/* Add the valid bit, maybe the caller was lazy. */
6076	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6077
6078	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6079	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6080	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6081
6082	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6083	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6084	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6085	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6086	AssertRCReturn(rc, rc);
6087
6088	return rc;
6089	}
6090
6091
6092	/**
6093	* Enters the VT-x session.
6094	*
6095	* @returns VBox status code.
6096	* @param pVM Pointer to the VM.
6097	* @param pVCpu Pointer to the VMCPU.
6098	* @param pCpu Pointer to the CPU info struct.
6099	*/
6100	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6101	{
6102	AssertPtr(pVM);
6103	AssertPtr(pVCpu);
6104	Assert(pVM->hm.s.vmx.fSupported);
6105	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6106	NOREF(pCpu);
6107
6108	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6109
6110	/* Make sure we're in VMX root mode. */
6111	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6112	if (!(u32HostCR4 & X86_CR4_VMXE))
6113	{
6114	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6115	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6116	}
6117
6118	/* Load the active VMCS as the current one. */
6119	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6120	if (RT_FAILURE(rc))
6121	return rc;
6122
6123	/** @todo this will change with preemption hooks where can can VMRESUME as long
6124	* as we're no preempted. */
6125	pVCpu->hm.s.fResumeVM = false;
6126	return VINF_SUCCESS;
6127	}
6128
6129
6130	/**
6131	* Leaves the VT-x session.
6132	*
6133	* @returns VBox status code.
6134	* @param pVM Pointer to the VM.
6135	* @param pVCpu Pointer to the VMCPU.
6136	* @param pCtx Pointer to the guest-CPU context.
6137	*/
6138	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6139	{
6140	AssertPtr(pVCpu);
6141	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6142	NOREF(pVM);
6143	NOREF(pCtx);
6144
6145	/** @todo this will change with preemption hooks where we only VMCLEAR when
6146	* we are actually going to be preempted, not all the time like we
6147	* currently do. */
6148	/*
6149	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6150	* and mark the VMCS launch-state as "clear".
6151	*/
6152	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6153	return rc;
6154	}
6155
6156
6157	/**
6158	* Saves the host state in the VMCS host-state.
6159	* Sets up the VM-exit MSR-load area.
6160	*
6161	* The CPU state will be loaded from these fields on every successful VM-exit.
6162	*
6163	* @returns VBox status code.
6164	* @param pVM Pointer to the VM.
6165	* @param pVCpu Pointer to the VMCPU.
6166	*
6167	* @remarks No-long-jump zone!!!
6168	*/
6169	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6170	{
6171	AssertPtr(pVM);
6172	AssertPtr(pVCpu);
6173	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6174
6175	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6176
6177	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6178	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6179	return VINF_SUCCESS;
6180
6181	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6182	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6183
6184	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6185	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6186
6187	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6188	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6189
6190	/* Reset the host-state-changed flag. */
6191	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6192	return rc;
6193	}
6194
6195
6196	/**
6197	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6198	* loaded from these fields on every successful VM-entry.
6199	*
6200	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6201	* Sets up the VM-entry controls.
6202	* Sets up the appropriate VMX non-root function to execute guest code based on
6203	* the guest CPU mode.
6204	*
6205	* @returns VBox status code.
6206	* @param pVM Pointer to the VM.
6207	* @param pVCpu Pointer to the VMCPU.
6208	* @param pCtx Pointer to the guest-CPU context.
6209	*
6210	* @remarks No-long-jump zone!!!
6211	*/
6212	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6213	{
6214	AssertPtr(pVM);
6215	AssertPtr(pVCpu);
6216	AssertPtr(pCtx);
6217	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6218
6219	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6220
6221	/* Currently we always atleast reload CR0 (longjmps included because of FPU state sharing). */
6222	Log(("LdGstFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6223
6224	/* Determine real-on-v86 mode. */
6225	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6226	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6227	&& CPUMIsGuestInRealModeEx(pCtx))
6228	{
6229	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6230	}
6231
6232	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6233	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6234
6235	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6236	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6237
6238	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6239	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6240
6241	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6242	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6243
6244	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6245	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6246
6247	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6248	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6249
6250	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6251	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6252
6253	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6254	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6255
6256	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6257	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6258
6259	rc = hmR0VmxLoadGuestIntrState(pVM, pVCpu, pCtx);
6260	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestIntrState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6261
6262	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6263	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6264
6265	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6266	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6267	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6268	return rc;
6269	}
6270
6271
6272	/**
6273	* Does the preparations before executing guest code in VT-x.
6274	*
6275	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6276	* recompiler. We must be cautious what we do here regarding committing
6277	* guest-state information into the the VMCS assuming we assuredly execute the
6278	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6279	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6280	* that the recompiler can (and should) use them when it resumes guest
6281	* execution. Otherwise such operations must be done when we can no longer
6282	* exit to ring-3.
6283	*
6284	* @returns VBox status code (informational status codes included).
6285	* @retval VINF_SUCCESS if we can proceed with running the guest.
6286	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6287	* into the guest.
6288	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6289	*
6290	* @param pVM Pointer to the VM.
6291	* @param pVCpu Pointer to the VMCPU.
6292	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6293	* out-of-sync. Make sure to update the required fields
6294	* before using them.
6295	* @param pVmxTransient Pointer to the VMX transient structure.
6296	*
6297	* @remarks Called with preemption disabled.
6298	*/
6299	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6300	{
6301	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6302
6303	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6304	PGMRZDynMapFlushAutoSet(pVCpu);
6305	#endif
6306
6307	/* Check force flag actions that might require us to go back to ring-3. */
6308	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6309	if (rc != VINF_SUCCESS)
6310	return rc;
6311
6312	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6313	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6314	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6315	{
6316	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6317	RTGCPHYS GCPhysApicBase;
6318	GCPhysApicBase = pMixedCtx->msrApicBase;
6319	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6320
6321	/* Unalias any existing mapping. */
6322	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6323	AssertRCReturn(rc, rc);
6324
6325	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6326	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6327	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6328	AssertRCReturn(rc, rc);
6329
6330	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6331	}
6332
6333	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6334	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6335	pVmxTransient->uEFlags = ASMIntDisableFlags();
6336	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6337	{
6338	ASMSetFlags(pVmxTransient->uEFlags);
6339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
6340	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6341	return VINF_EM_RAW_INTERRUPT;
6342	}
6343	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6344	#endif
6345
6346	/*
6347	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6348	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6349	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6350	*/
6351	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6352	AssertRCReturn(rc, rc);
6353	return rc;
6354	}
6355
6356
6357	/**
6358	* Prepares to run guest code in VT-x and we've committed to doing so. This
6359	* means there is no backing out to ring-3 or anywhere else at this
6360	* point.
6361	*
6362	* @param pVM Pointer to the VM.
6363	* @param pVCpu Pointer to the VMCPU.
6364	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6365	* out-of-sync. Make sure to update the required fields
6366	* before using them.
6367	* @param pVmxTransient Pointer to the VMX transient structure.
6368	*
6369	* @remarks Called with preemption disabled.
6370	* @remarks No-long-jump zone!!!
6371	*/
6372	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6373	{
6374	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6375
6376	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6377	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6378	pVmxTransient->uEFlags = ASMIntDisableFlags();
6379	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6380	#endif
6381
6382	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6383	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6384	int rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6385	AssertRC(rc);
6386	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6387
6388	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6389	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6390	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6391
6392	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6393	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6394	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6395
6396	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6397	if (pVmxTransient->fUpdateTscOffsetting)
6398	{
6399	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6400	pVmxTransient->fUpdateTscOffsetting = false;
6401	}
6402
6403	/*
6404	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6405	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6406	*/
6407	if (pVM->hm.s.fTPRPatchingActive)
6408	{
6409	Assert(!CPUMIsGuestInLongMode(pVCpu));
6410
6411	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6412	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6413	AssertRC(rc);
6414
6415	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6416	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6417	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6418	see hmR0VmxLoadGuestApicState(). */
6419	}
6420
6421	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6422	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6423	to start executing. */
6424	}
6425
6426
6427	/**
6428	* Performs some essential restoration of state after running guest code in
6429	* VT-x.
6430	*
6431	* @param pVM Pointer to the VM.
6432	* @param pVCpu Pointer to the VMCPU.
6433	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6434	* out-of-sync. Make sure to update the required fields
6435	* before using them.
6436	* @param pVmxTransient Pointer to the VMX transient structure.
6437	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6438	*
6439	* @remarks Called with interrupts disabled.
6440	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6441	* unconditionally when it is safe to do so.
6442	*/
6443	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6444	{
6445	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6446	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatInGC, x);
6447
6448	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6449	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6450	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6451	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6452	pVmxTransient->fVectoringPF = false; /* Clear the vectoring page-fault flag. */
6453
6454	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6455	{
6456	/** @todo Find a way to fix hardcoding a guestimate. */
6457	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6458	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6459	}
6460
6461	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6462	Assert(!(ASMGetFlags() & X86_EFL_IF));
6463	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6464
6465	/* Restore the effects of TPR patching if any. */
6466	if (pVM->hm.s.fTPRPatchingActive)
6467	{
6468	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6469	AssertRC(rc);
6470	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6471	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6472	}
6473
6474	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6475	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6476
6477	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6478	uint32_t uExitReason;
6479	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6480	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6481	AssertRC(rc);
6482	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6483	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6484
6485	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6486	hmR0VmxCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6487
6488	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6489	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6490	{
6491	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6492	return;
6493	}
6494
6495	/* We need to update our interruptibility-state on every VM-exit and VM-entry. */
6496	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6497	AssertRC(rc);
6498	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE;
6499
6500	/*
6501	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever we
6502	* eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why we do
6503	* it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6504	*/
6505	if ( !pVmxTransient->fVMEntryFailed
6506	&& (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6507	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6508	{
6509	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6510	AssertRC(rc);
6511	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6512	}
6513	}
6514
6515
6516	/**
6517	* Runs the guest code using VT-x.
6518	*
6519	* @returns VBox status code.
6520	* @param pVM Pointer to the VM.
6521	* @param pVCpu Pointer to the VMCPU.
6522	* @param pCtx Pointer to the guest-CPU context.
6523	*
6524	* @remarks Called with preemption disabled.
6525	*/
6526	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6527	{
6528	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6529	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6530	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6531
6532	VMXTRANSIENT VmxTransient;
6533	VmxTransient.fUpdateTscOffsetting = true;
6534	int rc = VERR_INTERNAL_ERROR_5;
6535	unsigned cLoops = 0;
6536
6537	for (;; cLoops++)
6538	{
6539	Assert(!HMR0SuspendPending());
6540	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6541	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6542	(unsigned)RTMpCpuId(), cLoops));
6543
6544	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6545	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6546	if (rc != VINF_SUCCESS)
6547	break;
6548
6549	/*
6550	* No longjmps to ring-3 from this point on!!!
6551	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6552	* This also disables flushing of the R0-logger instance (if any).
6553	*/
6554	hmR0VmxCallRing3Disable(pVCpu);
6555	VMMRZCallRing3RemoveNotification(pVCpu);
6556	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6557
6558	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6559	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6560
6561	/*
6562	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6563	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6564	*/
6565	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6566	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6567	{
6568	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6569	return rc;
6570	}
6571
6572	/* Handle any exception that caused a VM-exit while delivering an event to the guest. */
6573	rc = hmR0VmxSavePendingEventDueToEventDelivery(pVM, pVCpu, pCtx, &VmxTransient);
6574	if (RT_LIKELY(rc == VINF_SUCCESS))
6575	{
6576	/* Handle VM-exits. */
6577	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("Invalid VM-exit %#x\n", VmxTransient.uExitReason));
6578	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6579	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6580	if (rc != VINF_SUCCESS)
6581	break;
6582	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6583	{
6584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6585	rc = VINF_EM_RAW_INTERRUPT;
6586	break;
6587	}
6588	}
6589	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6590	break;
6591	/* else continue guest execution for (VINF_VMX_DOUBLE_FAULT) */
6592	}
6593
6594	if (rc == VERR_EM_INTERPRETER)
6595	rc = VINF_EM_RAW_EMULATE_INSTR;
6596	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6597	return rc;
6598	}
6599
6600	#ifdef DEBUG
6601	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6602	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6603	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6604	# define VMX_ASSERT_PREEMPT_CPUID() \
6605	do \
6606	{ \
6607	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6608	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6609	} while (0)
6610
6611	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6612	do { \
6613	AssertPtr(pVM); \
6614	AssertPtr(pVCpu); \
6615	AssertPtr(pMixedCtx); \
6616	AssertPtr(pVmxTransient); \
6617	Assert(pVmxTransient->fVMEntryFailed == false); \
6618	Assert(ASMIntAreEnabled() == true); \
6619	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6620	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6621	LogFunc(("\n")); \
6622	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6623	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6624	VMX_ASSERT_PREEMPT_CPUID(); \
6625	} while (0)
6626	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6627	do { \
6628	LogFunc(("\n")); \
6629	} while(0)
6630	#else /* Release builds */
6631	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6632	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6633	#endif
6634
6635
6636	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6637	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6638	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6639	/**
6640	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6641	*/
6642	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6643	{
6644	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6645	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6646	return VINF_EM_RAW_INTERRUPT;
6647	}
6648
6649
6650	/**
6651	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6652	*/
6653	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6654	{
6655	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6656	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6657	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6658	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6659	AssertRCReturn(rc, rc);
6660
6661	uint8_t u8IntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6662	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6663	&& u8IntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6664
6665	if (u8IntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6666	return VINF_EM_RAW_INTERRUPT;
6667
6668	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6669	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(uExitIntrInfo));
6670
6671	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6672	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo))
6673	{
6674	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6675	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6676	/* no break */
6677	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6678	{
6679	switch (uVector)
6680	{
6681	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6682	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6683	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6684	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6685	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6688	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6689	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6691	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6692	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6693	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6695	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6697	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	#endif
6699	default:
6700	{
6701	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6702	AssertRCReturn(rc, rc);
6703
6704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6705	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6706	{
6707	Assert(pVM->hm.s.vmx.pRealModeTSS);
6708	Assert(PDMVmmDevHeapIsEnabled(pVM));
6709	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6710	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6711	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6712	AssertRCReturn(rc, rc);
6713	}
6714	else
6715	{
6716	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6717	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6718	}
6719	break;
6720	}
6721	}
6722	break;
6723	}
6724
6725	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6726	default:
6727	{
6728	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6729	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6730	break;
6731	}
6732	}
6733	return rc;
6734	}
6735
6736
6737	/**
6738	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6739	*/
6740	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6741	{
6742	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6743	int rc = VERR_INTERNAL_ERROR_5;
6744	#ifdef DEBUG
6745	rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6746	AssertRC(rc);
6747	Assert(pMixedCtx->eflags.u32 & X86_EFL_IF);
6748	#endif
6749
6750	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6751	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6752	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6753	AssertRCReturn(rc, rc);
6754
6755	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6756	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6757	return VINF_SUCCESS;
6758	}
6759
6760
6761	/**
6762	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6763	*/
6764	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6765	{
6766	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6767	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6768	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6769	}
6770
6771
6772	/**
6773	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6774	*/
6775	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6776	{
6777	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6778	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6779	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6780	AssertRCReturn(rc, rc);
6781
6782	pMixedCtx->rip += pVmxTransient->cbInstr;
6783	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6784
6785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6786	return VINF_SUCCESS;
6787	}
6788
6789
6790	/**
6791	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6792	*/
6793	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6794	{
6795	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6796	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6797	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6798	AssertRCReturn(rc, rc);
6799
6800	pMixedCtx->rip += pVmxTransient->cbInstr;
6801	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6802
6803	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6804	return VINF_SUCCESS;
6805	}
6806
6807
6808	/**
6809	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6810	*/
6811	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6812	{
6813	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6814	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6815	if (RT_LIKELY(rc == VINF_SUCCESS))
6816	{
6817	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6818	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6819	AssertRCReturn(rc, rc);
6820	Assert(pVmxTransient->cbInstr == 2);
6821
6822	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6823	pMixedCtx->rip += pVmxTransient->cbInstr;
6824	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6825	}
6826	else
6827	{
6828	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6829	rc = VERR_EM_INTERPRETER;
6830	}
6831	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6832	return rc;
6833	}
6834
6835
6836	/**
6837	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6838	*/
6839	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6840	{
6841	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6842	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6843	AssertRCReturn(rc, rc);
6844
6845	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6846	return VINF_EM_RAW_EMULATE_INSTR;
6847
6848	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6849	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6850	}
6851
6852
6853	/**
6854	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6855	*/
6856	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6857	{
6858	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6859	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6860	AssertRCReturn(rc, rc);
6861
6862	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6863	if (RT_LIKELY(rc == VINF_SUCCESS))
6864	{
6865	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6866	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6867	AssertRCReturn(rc, rc);
6868	Assert(pVmxTransient->cbInstr == 2);
6869
6870	pMixedCtx->rip += pVmxTransient->cbInstr;
6871	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6872
6873	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6874	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6875	pVmxTransient->fUpdateTscOffsetting = true;
6876	}
6877	else
6878	{
6879	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6880	rc = VERR_EM_INTERPRETER;
6881	}
6882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6883	return rc;
6884	}
6885
6886
6887	/**
6888	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6889	*/
6890	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6891	{
6892	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6893	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6894	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6895	AssertRCReturn(rc, rc);
6896
6897	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6898	if (RT_LIKELY(rc == VINF_SUCCESS))
6899	{
6900	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6901	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6902	AssertRCReturn(rc, rc);
6903	Assert(pVmxTransient->cbInstr == 3);
6904
6905	pMixedCtx->rip += pVmxTransient->cbInstr;
6906	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6907
6908	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6909	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6910	pVmxTransient->fUpdateTscOffsetting = true;
6911	}
6912	else
6913	{
6914	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6915	rc = VERR_EM_INTERPRETER;
6916	}
6917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6918	return rc;
6919	}
6920
6921
6922	/**
6923	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6924	*/
6925	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6926	{
6927	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6928	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6929	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6930	AssertRCReturn(rc, rc);
6931
6932	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6933	if (RT_LIKELY(rc == VINF_SUCCESS))
6934	{
6935	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6936	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6937	AssertRCReturn(rc, rc);
6938	Assert(pVmxTransient->cbInstr == 2);
6939
6940	pMixedCtx->rip += pVmxTransient->cbInstr;
6941	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6942	}
6943	else
6944	{
6945	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6946	rc = VERR_EM_INTERPRETER;
6947	}
6948	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6949	return rc;
6950	}
6951
6952
6953	/**
6954	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6955	*/
6956	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6957	{
6958	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6959	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6960	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6961	AssertRCReturn(rc, rc);
6962
6963	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6964	rc = VBOXSTRICTRC_VAL(rc2);
6965	if (RT_LIKELY(rc == VINF_SUCCESS))
6966	{
6967	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6968	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6969	AssertRCReturn(rc, rc);
6970
6971	pMixedCtx->rip += pVmxTransient->cbInstr;
6972	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6973	}
6974	else
6975	{
6976	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6977	pVmxTransient->uExitQualification, rc));
6978	rc = VERR_EM_INTERPRETER;
6979	}
6980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6981	return rc;
6982	}
6983
6984
6985	/**
6986	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6987	*/
6988	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6989	{
6990	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6991	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6992	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6993	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6994	AssertRCReturn(rc, rc);
6995
6996	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6997	if (RT_LIKELY(rc == VINF_SUCCESS))
6998	{
6999	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7000	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7001	AssertRCReturn(rc, rc);
7002
7003	pMixedCtx->rip += pVmxTransient->cbInstr;
7004	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7005	}
7006	else
7007	{
7008	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7009	rc = VERR_EM_INTERPRETER;
7010	}
7011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7012	return rc;
7013	}
7014
7015
7016	/**
7017	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7018	*/
7019	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7020	{
7021	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7022	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7023	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7024	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7025	AssertRCReturn(rc, rc);
7026
7027	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7028	rc = VBOXSTRICTRC_VAL(rc2);
7029	if (RT_LIKELY( rc == VINF_SUCCESS
7030	\|\| rc == VINF_EM_HALT))
7031	{
7032	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7033	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7034	AssertRCReturn(rc3, rc3);
7035
7036	pMixedCtx->rip += pVmxTransient->cbInstr;
7037	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7038
7039	if ( rc == VINF_EM_HALT
7040	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7041	{
7042	rc = VINF_SUCCESS;
7043	}
7044	}
7045	else
7046	{
7047	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7048	rc = VERR_EM_INTERPRETER;
7049	}
7050	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7051	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7052	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7053	return rc;
7054	}
7055
7056
7057	/**
7058	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7059	*/
7060	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7061	{
7062	/*
7063	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7064	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7065	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7066	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7067	*/
7068	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7069	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7070	}
7071
7072
7073	/**
7074	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7075	*/
7076	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7077	{
7078	/*
7079	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7080	* root operation. If we get there there is something funny going on.
7081	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7082	*/
7083	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7084	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7085	}
7086
7087
7088	/**
7089	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7090	*/
7091	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7092	{
7093	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7094	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7095	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7096	}
7097
7098
7099	/**
7100	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7101	*/
7102	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7103	{
7104	/*
7105	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7106	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7107	* See Intel spec. 25.3 "Other Causes of VM-exits".
7108	*/
7109	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7110	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7111	}
7112
7113
7114	/**
7115	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7116	* VM-exit.
7117	*/
7118	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7119	{
7120	/*
7121	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7122	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7123	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7124	*/
7125	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7126	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7127	}
7128
7129
7130	/**
7131	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7132	* VM-exit.
7133	*/
7134	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7135	{
7136	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7137	return VINF_EM_RESET;
7138	}
7139
7140
7141	/**
7142	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7143	*/
7144	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7145	{
7146	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7147	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7148	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7149	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7150	AssertRCReturn(rc, rc);
7151
7152	pMixedCtx->rip++;
7153	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7154	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7155	rc = VINF_SUCCESS;
7156	else
7157	rc = VINF_EM_HALT;
7158
7159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7160	return rc;
7161	}
7162
7163
7164	/**
7165	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7166	*/
7167	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7168	{
7169	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7170	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7171	AssertRCReturn(rc, rc);
7172	return rc;
7173	}
7174
7175
7176	/**
7177	* VM-exit handler for expiry of the VMX preemption timer.
7178	*/
7179	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7180	{
7181	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7182	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7183	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7184	int rc = fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7185	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7186	return rc;
7187	}
7188
7189
7190	/**
7191	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7192	*/
7193	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7194	{
7195	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7196	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7197	/** @todo check if XSETBV is supported by the recompiler. */
7198	return VERR_EM_INTERPRETER;
7199	}
7200
7201
7202	/**
7203	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7204	*/
7205	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7206	{
7207	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7208	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7209	/** @todo implement EMInterpretInvpcid() */
7210	return VERR_EM_INTERPRETER;
7211	}
7212
7213
7214	/**
7215	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7216	* Error VM-exit.
7217	*/
7218	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7219	{
7220	uint32_t uIntrState;
7221	RTHCUINTREG uHCReg;
7222	uint64_t u64Val;
7223
7224	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7225	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7226	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7227	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7228	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7229	AssertRCReturn(rc, rc);
7230
7231	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7232	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7233	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7234	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7235
7236	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7237	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7238	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7239	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7240	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7241	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7242	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7243	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7244	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7245	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7246	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7247	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7248
7249	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7250
7251	return VERR_VMX_INVALID_GUEST_STATE;
7252	}
7253
7254
7255	/**
7256	* VM-exit handler for VM-entry failure due to an MSR-load
7257	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7258	*/
7259	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7260	{
7261	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7262	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7263	}
7264
7265
7266	/**
7267	* VM-exit handler for VM-entry failure due to a machine-check event
7268	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7269	*/
7270	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7271	{
7272	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7273	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7274	}
7275
7276
7277	/**
7278	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7279	* theory.
7280	*/
7281	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7282	{
7283	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7284	pVM, pVCpu, pMixedCtx));
7285	return VERR_VMX_UNDEFINED_EXIT_CODE;
7286	}
7287
7288
7289	/**
7290	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7291	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7292	* Conditional VM-exit.
7293	*/
7294	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7295	{
7296	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7297	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7299	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7300	return VERR_EM_INTERPRETER;
7301	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7302	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7303	}
7304
7305
7306	/**
7307	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7308	*/
7309	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7310	{
7311	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7312	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7314	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7315	return VERR_EM_INTERPRETER;
7316	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7317	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7318	}
7319
7320
7321	/**
7322	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7323	*/
7324	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7325	{
7326	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7327	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7328	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7329	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7330	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7331	AssertRCReturn(rc, rc);
7332
7333	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7334	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7335	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7337
7338	/* Update RIP and continue guest execution. */
7339	if (RT_LIKELY(rc == VINF_SUCCESS))
7340	{
7341	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7342	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7343	AssertRCReturn(rc, rc);
7344
7345	Assert(pVmxTransient->cbInstr == 2);
7346	pMixedCtx->rip += pVmxTransient->cbInstr;
7347	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7348	}
7349	return rc;
7350	}
7351
7352
7353	/**
7354	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7355	*/
7356	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7357	{
7358	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7359	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7360	AssertRCReturn(rc, rc);
7361	Assert(pVmxTransient->cbInstr == 2);
7362
7363	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7364	if ( pVM->hm.s.fTPRPatchingActive
7365	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7366	{
7367	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7368	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7369	{
7370	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7371	AssertRC(rc);
7372	}
7373
7374	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7375	AssertRCReturn(rc, rc);
7376	pMixedCtx->rip += pVmxTransient->cbInstr;
7377	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7378	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7379	return VINF_SUCCESS;
7380	}
7381
7382	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7383	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7384	{
7385	switch (pMixedCtx->ecx)
7386	{
7387	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7388	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7389	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7390	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7391	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7392	}
7393	}
7394	#ifdef DEBUG
7395	else
7396	{
7397	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7398	switch (pMixedCtx->ecx)
7399	{
7400	case MSR_IA32_SYSENTER_CS:
7401	case MSR_IA32_SYSENTER_EIP:
7402	case MSR_IA32_SYSENTER_ESP:
7403	case MSR_K8_FS_BASE:
7404	case MSR_K8_GS_BASE:
7405	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7406	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7407	case MSR_K8_LSTAR:
7408	case MSR_K6_STAR:
7409	case MSR_K8_SF_MASK:
7410	case MSR_K8_TSC_AUX:
7411	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7412	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7413	}
7414	}
7415	#endif
7416
7417	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7418	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7419	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7420	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7421	AssertRCReturn(rc, rc);
7422
7423	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7424	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7426
7427	/* Update guest-state and continue execution. */
7428	if (RT_LIKELY(rc == VINF_SUCCESS))
7429	{
7430	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7431	AssertRCReturn(rc, rc);
7432
7433	pMixedCtx->rip += pVmxTransient->cbInstr;
7434	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7435
7436	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7437	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7438	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7439	{
7440	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7441	}
7442	}
7443	return rc;
7444	}
7445
7446
7447	/**
7448	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7449	*/
7450	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7451	{
7452	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7453	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7455	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7456	return VERR_EM_INTERPRETER;
7457	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7458	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7459	}
7460
7461
7462	/**
7463	* VM-exit handler for when the TPR value is lowered below the specified
7464	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7465	*/
7466	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7467	{
7468	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7469	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7470
7471	/*
7472	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7473	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7474	* resume guest execution.
7475	*/
7476	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7478	return VINF_SUCCESS;
7479	}
7480
7481
7482	/**
7483	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7484	* VM-exit.
7485	*
7486	* @retval VINF_SUCCESS when guest execution can continue.
7487	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7488	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7489	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7490	* recompiler.
7491	*/
7492	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7493	{
7494	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7495	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7496	AssertRCReturn(rc, rc);
7497
7498	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7499	const uint8_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7500	switch (uAccessType)
7501	{
7502	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7503	{
7504	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7505	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7506	AssertRCReturn(rc, rc);
7507
7508	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7509	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7510	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7511	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7512
7513	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7514	{
7515	case 0: /* CR0 */
7516	Log(("CR0 write rc=%d\n", rc));
7517	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7518	break;
7519	case 2: /* CR2 */
7520	Log(("CR2 write rc=%d\n", rc));
7521	break;
7522	case 3: /* CR3 */
7523	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7524	Log(("CR3 write rc=%d\n", rc));
7525	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7526	break;
7527	case 4: /* CR4 */
7528	Log(("CR4 write rc=%d\n", rc));
7529	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7530	break;
7531	case 8: /* CR8 */
7532	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7533	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7534	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7535	break;
7536	default:
7537	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7538	break;
7539	}
7540
7541	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7542	break;
7543	}
7544
7545	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7546	{
7547	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7548	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7549	AssertRCReturn(rc, rc);
7550	Assert( !pVM->hm.s.fNestedPaging
7551	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7552	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7553
7554	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7555	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7556	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7557
7558	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7559	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7560	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7561	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7563	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7564	break;
7565	}
7566
7567	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7568	{
7569	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7570	rc \|= EMInterpretCLTS(pVM, pVCpu);
7571	AssertRCReturn(rc, rc);
7572	if (RT_LIKELY(rc == VINF_SUCCESS))
7573	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7574	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7575	Log(("CRX CLTS write rc=%d\n", rc));
7576	break;
7577	}
7578
7579	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7580	{
7581	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7582	rc \|= EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7583	AssertRCReturn(rc, rc);
7584	if (RT_LIKELY(rc == VINF_SUCCESS))
7585	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7587	Log(("CRX LMSW write rc=%d\n", rc));
7588	break;
7589	}
7590
7591	default:
7592	{
7593	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7594	return VERR_VMX_UNEXPECTED_EXCEPTION;
7595	}
7596	}
7597
7598	/* Validate possible error codes. */
7599	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7600	if (RT_SUCCESS(rc))
7601	{
7602	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7603	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7604	AssertRCReturn(rc2, rc2);
7605	pMixedCtx->rip += pVmxTransient->cbInstr;
7606	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7607	}
7608
7609	return rc;
7610	}
7611
7612
7613	/**
7614	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7615	* VM-exit.
7616	*/
7617	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7618	{
7619	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7620
7621	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7622	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7623	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7624	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7625	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7626	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7627	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7628	AssertRCReturn(rc, rc);
7629
7630	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7631	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7632	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7633	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7634	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7635	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7636	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7637
7638	/* I/O operation lookup arrays. */
7639	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7640	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7641
7642	const uint32_t cbSize = s_aIOSize[uIOWidth];
7643	const uint32_t cbInstr = pVmxTransient->cbInstr;
7644	if (fIOString)
7645	{
7646	/* INS/OUTS - I/O String instruction. */
7647	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7648	/** @todo for now manually disassemble later optimize by getting the fields from
7649	* the VMCS. */
7650	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7651	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7652	* segment prefix info. */
7653	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7654	if (RT_SUCCESS(rc))
7655	{
7656	if (fIOWrite)
7657	{
7658	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7659	(DISCPUMODE)pDis->uAddrMode, cbSize);
7660	rc = VBOXSTRICTRC_VAL(rc2);
7661	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7662	}
7663	else
7664	{
7665	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7666	(DISCPUMODE)pDis->uAddrMode, cbSize);
7667	rc = VBOXSTRICTRC_VAL(rc2);
7668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7669	}
7670	}
7671	else
7672	{
7673	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7674	rc = VINF_EM_RAW_EMULATE_INSTR;
7675	}
7676	}
7677	else
7678	{
7679	/* IN/OUT - I/O instruction. */
7680	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7681	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7682	if (fIOWrite)
7683	{
7684	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7685	rc = VBOXSTRICTRC_VAL(rc2);
7686	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7687	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7688	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7689	}
7690	else
7691	{
7692	uint32_t u32Result = 0;
7693	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7694	rc = VBOXSTRICTRC_VAL(rc2);
7695	if (IOM_SUCCESS(rc))
7696	{
7697	/* Save result of I/O IN instr. in AL/AX/EAX. */
7698	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7699	}
7700	else if (rc == VINF_IOM_R3_IOPORT_READ)
7701	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7703	}
7704	}
7705
7706	if (IOM_SUCCESS(rc))
7707	{
7708	pMixedCtx->rip += cbInstr;
7709	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7710	if (RT_LIKELY(rc == VINF_SUCCESS))
7711	{
7712	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7713	AssertRCReturn(rc, rc);
7714
7715	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7716	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7717	{
7718	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7719	for (unsigned i = 0; i < 4; i++)
7720	{
7721	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7722	if ( ( uIOPort >= pMixedCtx->dr[i]
7723	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7724	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7725	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7726	{
7727	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7728	uint64_t uDR6 = ASMGetDR6();
7729
7730	/* Clear all breakpoint status flags and set the one we just hit. */
7731	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7732	uDR6 \|= (uint64_t)RT_BIT(i);
7733
7734	/*
7735	* Note: AMD64 Architecture Programmer's Manual 13.1:
7736	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7737	* be cleared by software after the contents have been read.
7738	*/
7739	ASMSetDR6(uDR6);
7740
7741	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7742	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7743
7744	/* Paranoia. */
7745	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7746	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7747	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7748
7749	/* Resync DR7 */
7750	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7751	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7752
7753	/* Inject #DB and get on with guest execution. */
7754	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7755	AssertRCReturn(rc, rc);
7756	break;
7757	}
7758	}
7759	}
7760	}
7761	}
7762
7763	#ifdef DEBUG
7764	if (rc == VINF_IOM_R3_IOPORT_READ)
7765	Assert(!fIOWrite);
7766	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7767	Assert(fIOWrite);
7768	else
7769	{
7770	AssertMsg( RT_FAILURE(rc)
7771	\|\| rc == VINF_SUCCESS
7772	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7773	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7774	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7775	}
7776	#endif
7777
7778	return rc;
7779	}
7780
7781
7782	/**
7783	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7784	* VM-exit.
7785	*/
7786	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7787	{
7788	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7789
7790	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7791	* emulation. */
7792
7793	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7794	return VERR_EM_INTERPRETER;
7795	}
7796
7797
7798	/**
7799	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7800	*/
7801	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7802	{
7803	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7804	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7805	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7806	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7807	AssertRCReturn(rc, rc);
7808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7809	return VINF_EM_DBG_STOP;
7810	}
7811
7812
7813	/**
7814	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7815	*/
7816	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7817	{
7818	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7819	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7820
7821	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7822	* just sync the whole thing. */
7823	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7824	AssertRCReturn(rc, rc);
7825
7826	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7827	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7828	switch (uAccessType)
7829	{
7830	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7831	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7832	{
7833	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7834	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7835	{
7836	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7837	}
7838
7839	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7840	GCPhys &= PAGE_BASE_GC_MASK;
7841	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7842	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7843	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7844	CPUMCTX2CORE(pMixedCtx), GCPhys);
7845	rc = VBOXSTRICTRC_VAL(rc2);
7846	if ( rc == VINF_SUCCESS
7847	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7848	\|\| rc == VERR_PAGE_NOT_PRESENT)
7849	{
7850	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7851	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7852	rc = VINF_SUCCESS;
7853	}
7854	break;
7855	}
7856
7857	default:
7858	rc = VINF_EM_RAW_EMULATE_INSTR;
7859	break;
7860	}
7861
7862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7863	return rc;
7864	}
7865
7866
7867	/**
7868	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7869	* VM-exit.
7870	*/
7871	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7872	{
7873	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7874
7875	/* We should -not- get this VM-exit if the guest is debugging. */
7876	if (CPUMIsGuestDebugStateActive(pVCpu))
7877	{
7878	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7879	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7880	}
7881
7882	int rc = VERR_INTERNAL_ERROR_5;
7883	if ( !DBGFIsStepping(pVCpu)
7884	&& !CPUMIsHyperDebugStateActive(pVCpu))
7885	{
7886	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7887
7888	/* Don't intercept MOV DRx. */
7889	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7890	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7891	AssertRCReturn(rc, rc);
7892
7893	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7894	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7895	AssertRCReturn(rc, rc);
7896	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7897
7898	#ifdef VBOX_WITH_STATISTICS
7899	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7900	AssertRCReturn(rc, rc);
7901	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7902	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7903	else
7904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7905	#endif
7906	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7907	return VINF_SUCCESS;
7908	}
7909
7910	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7911	* time and restore DRx registers afterwards */
7912	/*
7913	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7914	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7915	*/
7916	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7917	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7918	AssertRCReturn(rc, rc);
7919
7920	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7921	{
7922	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7923	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7924	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7925	if (RT_SUCCESS(rc))
7926	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7928	}
7929	else
7930	{
7931	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7932	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7933	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7935	}
7936
7937	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7938	if (RT_SUCCESS(rc))
7939	{
7940	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7941	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7942	AssertRCReturn(rc2, rc2);
7943	pMixedCtx->rip += pVmxTransient->cbInstr;
7944	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7945	}
7946	return rc;
7947	}
7948
7949
7950	/**
7951	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7952	* Conditional VM-exit.
7953	*/
7954	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7955	{
7956	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7957	Assert(pVM->hm.s.fNestedPaging);
7958
7959	RTGCPHYS GCPhys = 0;
7960	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
7961	AssertRCReturn(rc, rc);
7962
7963	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
7964	AssertRCReturn(rc, rc);
7965
7966	Log(("EPT misconfig at %#RX64 RIP=%#RX64\n", GCPhys, pMixedCtx->rip));
7967
7968	/*
7969	* If we succeed, resume guest execution.
7970	* If we fail in interpreting the instruction because we couldn't get the guest physical address
7971	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
7972	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
7973	* weird case. See @bugref{6043}.
7974	*/
7975	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
7976	rc = VBOXSTRICTRC_VAL(rc2);
7977	Log(("EPT misconfig rc=%d\n", rc));
7978	if ( rc == VINF_SUCCESS
7979	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7980	\|\| rc == VERR_PAGE_NOT_PRESENT)
7981	{
7982	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7983	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7984	return VINF_SUCCESS;
7985	}
7986	return rc;
7987	}
7988
7989
7990	/**
7991	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
7992	* VM-exit.
7993	*/
7994	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7995	{
7996	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7997	Assert(pVM->hm.s.fNestedPaging);
7998
7999	RTGCPHYS GCPhys = 0;
8000	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8001	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8002	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo can we do better? */
8003	AssertRCReturn(rc, rc);
8004
8005	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8006	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8007
8008	RTGCUINT uErrorCode = 0;
8009	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8010	uErrorCode \|= X86_TRAP_PF_ID;
8011	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8012	uErrorCode \|= X86_TRAP_PF_RW;
8013	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8014	uErrorCode \|= X86_TRAP_PF_P;
8015
8016	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8017	TRPMSetErrorCode(pVCpu, uErrorCode);
8018	TRPMSetFaultAddress(pVCpu, GCPhys);
8019
8020	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8021	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8022
8023	/* Handle the pagefault trap for the nested shadow table. */
8024	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8025	TRPMResetTrap(pVCpu);
8026
8027	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8028	if ( rc == VINF_SUCCESS
8029	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8030	\|\| rc == VERR_PAGE_NOT_PRESENT)
8031	{
8032	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8034	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8035	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8036	return VINF_SUCCESS;
8037	}
8038
8039	Log(("EPT return to ring-3 rc=%d\n"));
8040
8041	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8042	/** @todo Shouldn't we update TRPM here? */
8043	return rc;
8044	}
8045
8046
8047	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8048	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8049	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8050	/**
8051	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8052	*/
8053	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8054	{
8055	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8056
8057	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8058	AssertRCReturn(rc, rc);
8059	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8060
8061	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8062	{
8063	/* Old-style FPU error reporting needs some extra work. */
8064	/** @todo don't fall back to the recompiler, but do it manually. */
8065	return VERR_EM_INTERPRETER;
8066	}
8067	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8068	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8069	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8070	AssertRCReturn(rc, rc);
8071	return rc;
8072	}
8073
8074
8075	/**
8076	* VM-exit exception handler for #BP (Breakpoint exception).
8077	*/
8078	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8079	{
8080	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8081
8082	/** @todo Try optimize this by not saving the entire guest state unless
8083	* really needed. */
8084	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8085	AssertRCReturn(rc, rc);
8086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8087
8088	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8089	if (rc == VINF_EM_RAW_GUEST_TRAP)
8090	{
8091	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8092	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8093	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8094	AssertRCReturn(rc, rc);
8095
8096	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8097	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8098	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8099	AssertRCReturn(rc, rc);
8100	}
8101
8102	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8103	return rc;
8104	}
8105
8106
8107	/**
8108	* VM-exit exception handler for #DB (Debug exception).
8109	*/
8110	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8111	{
8112	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8113
8114	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8115	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8116	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8117	AssertRCReturn(rc, rc);
8118
8119	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8120	uint64_t uDR6 = X86_DR6_INIT_VAL;
8121	uDR6 \|= (pVmxTransient->uExitQualification
8122	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8123	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8124	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8125	if (rc == VINF_EM_RAW_GUEST_TRAP)
8126	{
8127	/** @todo revisit this. */
8128	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8129	pMixedCtx->dr[6] = uDR6;
8130
8131	if (CPUMIsGuestDebugStateActive(pVCpu))
8132	ASMSetDR6(pMixedCtx->dr[6]);
8133
8134	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8135	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8136
8137	/* Paranoia. */
8138	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8139	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8140	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8141
8142	/* Resync DR7. */
8143	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8144
8145	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8146	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8147	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8148	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8149	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8150	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8151	AssertRCReturn(rc,rc);
8152	return rc;
8153	}
8154	/* Return to ring 3 to deal with the debug exit code. */
8155	return rc;
8156	}
8157
8158
8159	/**
8160	* VM-exit exception handler for #NM (Device-not-available exception: floating
8161	* point exception).
8162	*/
8163	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8164	{
8165	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8166
8167	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8168	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8169	#endif
8170
8171	/* We require CR0 and EFER. EFER is always up-to-date. */
8172	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8173	AssertRCReturn(rc, rc);
8174
8175	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8176	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8177	if (rc == VINF_SUCCESS)
8178	{
8179	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8180
8181	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8183	return VINF_SUCCESS;
8184	}
8185
8186	/* Forward #NM to the guest. */
8187	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8188	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8189	AssertRCReturn(rc, rc);
8190	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8191	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8192	pVmxTransient->cbInstr, 0 /* error code */);
8193	AssertRCReturn(rc, rc);
8194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8195	return rc;
8196	}
8197
8198
8199	/**
8200	* VM-exit exception handler for #GP (General-protection exception).
8201	*
8202	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8203	*/
8204	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8205	{
8206	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8207
8208	int rc = VERR_INTERNAL_ERROR_5;
8209
8210	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8211	{
8212	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8213	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8214	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8215	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8216	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8217	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8218	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8219	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8220	AssertRCReturn(rc, rc);
8221	return rc;
8222	#else
8223	/* We don't intercept #GP. */
8224	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8225	return VERR_VMX_UNEXPECTED_EXCEPTION;
8226	#endif
8227	}
8228
8229	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8230	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8231
8232	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8233	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8234	AssertRCReturn(rc, rc);
8235
8236	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8237	unsigned int cbOp = 0;
8238	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8239	if (RT_SUCCESS(rc))
8240	{
8241	rc = VINF_SUCCESS;
8242	Assert(cbOp == pDis->cbInstr);
8243	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8244	switch (pDis->pCurInstr->uOpcode)
8245	{
8246	case OP_CLI:
8247	pMixedCtx->eflags.Bits.u1IF = 0;
8248	pMixedCtx->rip += pDis->cbInstr;
8249	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8250	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8251	break;
8252
8253	case OP_STI:
8254	pMixedCtx->eflags.Bits.u1IF = 1;
8255	pMixedCtx->rip += pDis->cbInstr;
8256	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8257	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8258	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE \| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8260	break;
8261
8262	case OP_HLT:
8263	rc = VINF_EM_HALT;
8264	pMixedCtx->rip += pDis->cbInstr;
8265	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8267	break;
8268
8269	case OP_POPF:
8270	{
8271	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8272	size_t cbParm = 0;
8273	uint32_t uMask = 0;
8274	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8275	{
8276	cbParm = 4;
8277	uMask = 0xffffffff;
8278	}
8279	else
8280	{
8281	cbParm = 2;
8282	uMask = 0xffff;
8283	}
8284
8285	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8286	RTGCPTR GCPtrStack = 0;
8287	X86EFLAGS uEflags;
8288	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8289	&GCPtrStack);
8290	if (RT_SUCCESS(rc))
8291	{
8292	Assert(sizeof(uEflags.u32) >= cbParm);
8293	uEflags.u32 = 0;
8294	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8295	}
8296	if (RT_FAILURE(rc))
8297	{
8298	rc = VERR_EM_INTERPRETER;
8299	break;
8300	}
8301	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8302	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8303	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8304	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8305	pMixedCtx->eflags.Bits.u1RF = 0;
8306	pMixedCtx->esp += cbParm;
8307	pMixedCtx->esp &= uMask;
8308	pMixedCtx->rip += pDis->cbInstr;
8309	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8310	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8311	break;
8312	}
8313
8314	case OP_PUSHF:
8315	{
8316	size_t cbParm = 0;
8317	uint32_t uMask = 0;
8318	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8319	{
8320	cbParm = 4;
8321	uMask = 0xffffffff;
8322	}
8323	else
8324	{
8325	cbParm = 2;
8326	uMask = 0xffff;
8327	}
8328
8329	/* Get the stack pointer & push the contents of eflags onto the stack. */
8330	RTGCPTR GCPtrStack = 0;
8331	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8332	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8333	if (RT_FAILURE(rc))
8334	{
8335	rc = VERR_EM_INTERPRETER;
8336	break;
8337	}
8338	X86EFLAGS uEflags;
8339	uEflags = pMixedCtx->eflags;
8340	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8341	uEflags.Bits.u1RF = 0;
8342	uEflags.Bits.u1VM = 0;
8343
8344	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8345	if (RT_FAILURE(rc))
8346	{
8347	rc = VERR_EM_INTERPRETER;
8348	break;
8349	}
8350	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8351	pMixedCtx->esp -= cbParm;
8352	pMixedCtx->esp &= uMask;
8353	pMixedCtx->rip += pDis->cbInstr;
8354	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8355	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8356	break;
8357	}
8358
8359	case OP_IRET:
8360	{
8361	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8362	* instruction reference. */
8363	RTGCPTR GCPtrStack = 0;
8364	uint32_t uMask = 0xffff;
8365	uint16_t aIretFrame[3];
8366	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8367	{
8368	rc = VERR_EM_INTERPRETER;
8369	break;
8370	}
8371	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8372	&GCPtrStack);
8373	if (RT_SUCCESS(rc))
8374	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8375	if (RT_FAILURE(rc))
8376	{
8377	rc = VERR_EM_INTERPRETER;
8378	break;
8379	}
8380	pMixedCtx->eip = 0;
8381	pMixedCtx->ip = aIretFrame[0];
8382	pMixedCtx->cs.Sel = aIretFrame[1];
8383	pMixedCtx->cs.ValidSel = aIretFrame[1];
8384	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8385	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8386	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8387	pMixedCtx->sp += sizeof(aIretFrame);
8388	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8389	\| HM_CHANGED_GUEST_RFLAGS;
8390	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8392	break;
8393	}
8394
8395	case OP_INT:
8396	{
8397	uint16_t uVector = pDis->Param1.uValue & 0xff;
8398	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8399	AssertRCReturn(rc, rc);
8400	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8401	break;
8402	}
8403
8404	case OP_INTO:
8405	{
8406	if (pMixedCtx->eflags.Bits.u1OF)
8407	{
8408	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8410	}
8411	break;
8412	}
8413
8414	default:
8415	{
8416	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8417	EMCODETYPE_SUPERVISOR);
8418	rc = VBOXSTRICTRC_VAL(rc2);
8419	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8420	Log2(("#GP rc=%Rrc\n", rc));
8421	break;
8422	}
8423	}
8424	}
8425	else
8426	rc = VERR_EM_INTERPRETER;
8427
8428	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8429	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8430	("#GP Unexpected rc=%Rrc\n", rc));
8431	return rc;
8432	}
8433
8434
8435	/**
8436	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8437	* the exception reported in the VMX transient structure back into the VM.
8438	*
8439	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8440	* VMX transient structure to be up-to-date.
8441	*/
8442	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8443	{
8444	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8445
8446	/* Re-inject the exception into the guest. This cannot be a double-fault condition which are handled in
8447	hmR0VmxSavePendingEventDueToEventDelivery(). */
8448	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8449	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8450	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8451	AssertRCReturn(rc, rc);
8452	return rc;
8453	}
8454
8455
8456	/**
8457	* VM-exit exception handler for #PF (Page-fault exception).
8458	*/
8459	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8460	{
8461	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8462
8463	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8464	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8465	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8466	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8467	AssertRCReturn(rc, rc);
8468
8469	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8470	if (pVM->hm.s.fNestedPaging)
8471	{
8472	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8473	{
8474	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8475	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8476	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8477	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8478	AssertRCReturn(rc, rc);
8479	}
8480	else
8481	{
8482	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8483	Assert(!pVCpu->hm.s.Event.fPending);
8484	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8485	}
8486	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8487	return rc;
8488	}
8489	#else
8490	Assert(!pVM->hm.s.fNestedPaging);
8491	#endif
8492
8493	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8494	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8495	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8496	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8497	AssertRCReturn(rc, rc);
8498	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8499	if ( pVM->hm.s.fTRPPatchingAllowed
8500	&& pVM->hm.s.pGuestPatchMem
8501	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8502	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8503	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8504	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8505	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8506	{
8507	RTGCPHYS GCPhys;
8508	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8509	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8510	if ( rc == VINF_SUCCESS
8511	&& GCPhys == GCPhysApicBase)
8512	{
8513	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8514	AssertRCReturn(rc, rc);
8515
8516	/* Only attempt to patch the instruction once. */
8517	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8518	if (!pPatch)
8519	return VINF_EM_HM_PATCH_TPR_INSTR;
8520	}
8521	}
8522	#endif
8523
8524	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8525	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8526	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8527
8528	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8529	AssertRCReturn(rc, rc);
8530
8531	/* Forward it to the trap handler first. */
8532	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8533	(RTGCPTR)pVmxTransient->uExitQualification);
8534
8535	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8536	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8537
8538	if (rc == VINF_SUCCESS)
8539	{
8540	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8541	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8542	* memory? We don't update the whole state here... */
8543	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8544	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8545
8546	TRPMResetTrap(pVCpu);
8547	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8548	return rc;
8549	}
8550	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8551	{
8552	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8553	{
8554	/* It's a guest page fault and needs to be reflected to the guest. */
8555	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8556	TRPMResetTrap(pVCpu);
8557	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8558	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8559	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8560	pVmxTransient->cbInstr, uGstErrorCode);
8561	AssertRCReturn(rc, rc);
8562	}
8563	else
8564	{
8565	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8566	Assert(!pVCpu->hm.s.Event.fPending);
8567	TRPMResetTrap(pVCpu);
8568	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8569	}
8570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8571	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8572	return rc;
8573	}
8574
8575	TRPMResetTrap(pVCpu);
8576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8577	return rc;
8578	}
8579

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45378

Download in other formats: