HMVMXR0.cpp@ 45639

Last change on this file since 45639 was 45639, checked in by vboxsync, 12 years ago
VMMR0/HMVMXR0: Simultaneous hyper and guest single-stepping support when there is no Monitor Trap Flag capability on the CPU.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.9 KB

Line
1	/* $Id: HMVMXR0.cpp 45639 2013-04-19 12:09:02Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_TRAP_ALL_EXCEPTIONS
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define HMVMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define HMVMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/** Use the function table. */
56	#define HMVMX_USE_FUNCTION_TABLE
57
58	/** This bit indicates the segment selector is unusable in VT-x. */
59	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
60
61	/** Determine which tagged-TLB flush handler to use. */
62	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67	/**
68	* Updated-guest-state flags.
69	*/
70	#define HMVMX_UPDATED_GUEST_FPU RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_TR RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
91	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_FPU \
92	\| HMVMX_UPDATED_GUEST_RIP \
93	\| HMVMX_UPDATED_GUEST_RSP \
94	\| HMVMX_UPDATED_GUEST_RFLAGS \
95	\| HMVMX_UPDATED_GUEST_CR0 \
96	\| HMVMX_UPDATED_GUEST_CR3 \
97	\| HMVMX_UPDATED_GUEST_CR4 \
98	\| HMVMX_UPDATED_GUEST_GDTR \
99	\| HMVMX_UPDATED_GUEST_IDTR \
100	\| HMVMX_UPDATED_GUEST_LDTR \
101	\| HMVMX_UPDATED_GUEST_TR \
102	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
103	\| HMVMX_UPDATED_GUEST_DEBUG \
104	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
105	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
109	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
110	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
111	\| HMVMX_UPDATED_GUEST_APIC_STATE)
112
113	/**
114	* Flags to skip redundant reads of some common VMCS fields that are not part of
115	* the guest-CPU state but are in the transient structure.
116	*/
117	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
123
124	/**
125	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
126	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
127	* we have Nested Paging support.
128	*/
129	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
130	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
131	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
132	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
133	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
134	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
135	\| RT_BIT(X86_XCPT_XF))
136
137	/**
138	* Exception bitmap mask for all contributory exceptions.
139	*/
140	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
141	\| RT_BIT(X86_XCPT_DE))
142
143	/** Maximum VM-instruction error number. */
144	#define HMVMX_INSTR_ERROR_MAX 28
145
146
147	/*******************************************************************************
148	* Structures and Typedefs *
149	*******************************************************************************/
150	/**
151	* A state structure for holding miscellaneous information across
152	* VMX non-root operation and restored after the transition.
153	*/
154	typedef struct VMXTRANSIENT
155	{
156	/** The host's rflags/eflags. */
157	RTCCUINTREG uEFlags;
158	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
159	uint64_t u64LStarMsr;
160	/** The guest's TPR value used for TPR shadowing. */
161	uint8_t u8GuestTpr;
162
163	/** The basic VM-exit reason. */
164	uint16_t uExitReason;
165	/** The VM-exit exit qualification. */
166	RTGCUINTPTR uExitQualification;
167	/** The VM-exit interruption error code. */
168	uint32_t uExitIntrErrorCode;
169
170	/** The VM-exit interruption-information field. */
171	uint32_t uExitIntrInfo;
172	/** Whether the VM-entry failed or not. */
173	bool fVMEntryFailed;
174	/** The VM-exit instruction-length field. */
175	uint32_t cbInstr;
176
177	/** The VM-entry interruption-information field. */
178	uint32_t uEntryIntrInfo;
179	/** The VM-entry exception error code field. */
180	uint32_t uEntryXcptErrorCode;
181	/** The VM-entry instruction length field. */
182	uint32_t cbEntryInstr;
183
184	/** IDT-vectoring information field. */
185	uint32_t uIdtVectoringInfo;
186	/** IDT-vectoring error code. */
187	uint32_t uIdtVectoringErrorCode;
188
189	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
190	uint32_t fVmcsFieldsRead;
191	/** Whether TSC-offsetting should be setup before VM-entry. */
192	bool fUpdateTscOffsettingAndPreemptTimer;
193	/** Whether the VM-exit was caused by a page-fault during delivery of a
194	* contributary exception or a page-fault. */
195	bool fVectoringPF;
196	} VMXTRANSIENT, *PVMXTRANSIENT;
197
198	/**
199	* MSR-bitmap read permissions.
200	*/
201	typedef enum VMXMSREXITREAD
202	{
203	/** Reading this MSR causes a VM-exit. */
204	VMXMSREXIT_INTERCEPT_READ = 0xb,
205	/** Reading this MSR does not cause a VM-exit. */
206	VMXMSREXIT_PASSTHRU_READ
207	} VMXMSREXITREAD;
208
209	/**
210	* MSR-bitmap write permissions.
211	*/
212	typedef enum VMXMSREXITWRITE
213	{
214	/** Writing to this MSR causes a VM-exit. */
215	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
216	/** Writing to this MSR does not cause a VM-exit. */
217	VMXMSREXIT_PASSTHRU_WRITE
218	} VMXMSREXITWRITE;
219
220
221	/*******************************************************************************
222	* Internal Functions *
223	*******************************************************************************/
224	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
225	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
226	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
227	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
228	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
229	#endif
230	#ifndef HMVMX_USE_FUNCTION_TABLE
231	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
232	#endif
233
234	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278
279	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286
287
288	/*******************************************************************************
289	* Global Variables *
290	*******************************************************************************/
291	#ifdef HMVMX_USE_FUNCTION_TABLE
292	/**
293	* VM-exit handler.
294	*
295	* @returns VBox status code.
296	* @param pVCpu Pointer to the VMCPU.
297	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
298	* out-of-sync. Make sure to update the required
299	* fields before using them.
300	* @param pVmxTransient Pointer to the VMX-transient structure.
301	*/
302	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	/** Pointer to VM-exit handler. */
304	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
305
306	/**
307	* VMX_EXIT dispatch table.
308	*/
309	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
310	{
311	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
312	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
313	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
314	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
315	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
316	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
317	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
318	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
319	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
320	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
321	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
322	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
323	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
324	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
325	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
326	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
327	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
328	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
329	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
330	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
331	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
332	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
333	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
334	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
335	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
336	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
337	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
338	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
339	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
340	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
341	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
342	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
343	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
344	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
345	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
346	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
347	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
348	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
349	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
350	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
351	/* 40 UNDEFINED */ hmR0VmxExitPause,
352	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
353	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
354	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
355	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
356	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
357	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
358	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
359	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
360	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
361	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
362	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
363	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
364	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
365	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
366	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
367	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
368	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
369	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
370	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
371	};
372
373	#endif /* HMVMX_USE_FUNCTION_TABLE */
374
375	#ifdef VBOX_STRICT
376	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
377	{
378	/* 0 */ "(Not Used)",
379	/* 1 */ "VMCALL executed in VMX root operation.",
380	/* 2 */ "VMCLEAR with invalid physical address.",
381	/* 3 */ "VMCLEAR with VMXON pointer.",
382	/* 4 */ "VMLAUNCH with non-clear VMCS.",
383	/* 5 */ "VMRESUME with non-launched VMCS.",
384	/* 6 */ "VMRESUME after VMXOFF",
385	/* 7 */ "VM entry with invalid control fields.",
386	/* 8 */ "VM entry with invalid host state fields.",
387	/* 9 */ "VMPTRLD with invalid physical address.",
388	/* 10 */ "VMPTRLD with VMXON pointer.",
389	/* 11 */ "VMPTRLD with incorrect revision identifier.",
390	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
391	/* 13 */ "VMWRITE to read-only VMCS component.",
392	/* 14 */ "(Not Used)",
393	/* 15 */ "VMXON executed in VMX root operation.",
394	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
395	/* 17 */ "VM entry with non-launched executing VMCS.",
396	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
397	/* 19 */ "VMCALL with non-clear VMCS.",
398	/* 20 */ "VMCALL with invalid VM-exit control fields.",
399	/* 21 */ "(Not Used)",
400	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
401	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
402	/* 24 */ "VMCALL with invalid SMM-monitor features.",
403	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
404	/* 26 */ "VM entry with events blocked by MOV SS.",
405	/* 27 */ "(Not Used)",
406	/* 28 */ "Invalid operand to INVEPT/INVVPID."
407	};
408	#endif /* VBOX_STRICT */
409
410
411
412	/**
413	* Updates the VM's last error record. If there was a VMX instruction error,
414	* reads the error data from the VMCS and updates VCPU's last error record as
415	* well.
416	*
417	* @param pVM Pointer to the VM.
418	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
419	* VERR_VMX_UNABLE_TO_START_VM or
420	* VERR_VMX_INVALID_VMCS_FIELD).
421	* @param rc The error code.
422	*/
423	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
424	{
425	AssertPtr(pVM);
426	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
427	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
428	{
429	AssertPtrReturnVoid(pVCpu);
430	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
431	}
432	pVM->hm.s.lLastError = rc;
433	}
434
435
436	/**
437	* Reads the VM-entry interruption-information field from the VMCS into the VMX
438	* transient structure.
439	*
440	* @returns VBox status code.
441	* @param pVmxTransient Pointer to the VMX transient structure.
442	*
443	* @remarks No-long-jump zone!!!
444	*/
445	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
446	{
447	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
448	AssertRCReturn(rc, rc);
449	return VINF_SUCCESS;
450	}
451
452
453	/**
454	* Reads the VM-entry exception error code field from the VMCS into
455	* the VMX transient structure.
456	*
457	* @returns VBox status code.
458	* @param pVmxTransient Pointer to the VMX transient structure.
459	*
460	* @remarks No-long-jump zone!!!
461	*/
462	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
463	{
464	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
465	AssertRCReturn(rc, rc);
466	return VINF_SUCCESS;
467	}
468
469
470	/**
471	* Reads the VM-entry exception error code field from the VMCS into
472	* the VMX transient structure.
473	*
474	* @returns VBox status code.
475	* @param pVCpu Pointer to the VMCPU.
476	* @param pVmxTransient Pointer to the VMX transient structure.
477	*
478	* @remarks No-long-jump zone!!!
479	*/
480	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
481	{
482	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
483	AssertRCReturn(rc, rc);
484	return VINF_SUCCESS;
485	}
486
487
488	/**
489	* Reads the VM-exit interruption-information field from the VMCS into the VMX
490	* transient structure.
491	*
492	* @returns VBox status code.
493	* @param pVCpu Pointer to the VMCPU.
494	* @param pVmxTransient Pointer to the VMX transient structure.
495	*/
496	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
497	{
498	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
499	{
500	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
501	AssertRCReturn(rc, rc);
502	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
503	}
504	return VINF_SUCCESS;
505	}
506
507
508	/**
509	* Reads the VM-exit interruption error code from the VMCS into the VMX
510	* transient structure.
511	*
512	* @returns VBox status code.
513	* @param pVCpu Pointer to the VMCPU.
514	* @param pVmxTransient Pointer to the VMX transient structure.
515	*/
516	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
517	{
518	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
519	{
520	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
521	AssertRCReturn(rc, rc);
522	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
523	}
524	return VINF_SUCCESS;
525	}
526
527
528	/**
529	* Reads the VM-exit instruction length field from the VMCS into the VMX
530	* transient structure.
531	*
532	* @returns VBox status code.
533	* @param pVCpu Pointer to the VMCPU.
534	* @param pVmxTransient Pointer to the VMX transient structure.
535	*/
536	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
537	{
538	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
539	{
540	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
541	AssertRCReturn(rc, rc);
542	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
543	}
544	return VINF_SUCCESS;
545	}
546
547
548	/**
549	* Reads the exit qualification from the VMCS into the VMX transient structure.
550	*
551	* @returns VBox status code.
552	* @param pVCpu Pointer to the VMCPU.
553	* @param pVmxTransient Pointer to the VMX transient structure.
554	*/
555	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
556	{
557	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
558	{
559	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
560	AssertRCReturn(rc, rc);
561	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
562	}
563	return VINF_SUCCESS;
564	}
565
566
567	/**
568	* Reads the IDT-vectoring information field from the VMCS into the VMX
569	* transient structure.
570	*
571	* @returns VBox status code.
572	* @param pVmxTransient Pointer to the VMX transient structure.
573	*
574	* @remarks No-long-jump zone!!!
575	*/
576	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
577	{
578	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
579	{
580	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
581	AssertRCReturn(rc, rc);
582	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
583	}
584	return VINF_SUCCESS;
585	}
586
587
588	/**
589	* Reads the IDT-vectoring error code from the VMCS into the VMX
590	* transient structure.
591	*
592	* @returns VBox status code.
593	* @param pVmxTransient Pointer to the VMX transient structure.
594	*/
595	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
596	{
597	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
598	{
599	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
600	AssertRCReturn(rc, rc);
601	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
602	}
603	return VINF_SUCCESS;
604	}
605
606
607	/**
608	* Enters VMX root mode operation on the current CPU.
609	*
610	* @returns VBox status code.
611	* @param pVM Pointer to the VM (optional, can be NULL, after
612	* a resume).
613	* @param HCPhysCpuPage Physical address of the VMXON region.
614	* @param pvCpuPage Pointer to the VMXON region.
615	*/
616	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
617	{
618	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
619	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
620
621	if (pVM)
622	{
623	/* Write the VMCS revision dword to the VMXON region. */
624	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
625	}
626
627	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
628	RTCCUINTREG fFlags = ASMIntDisableFlags();
629
630	/* Enable the VMX bit in CR4 if necessary. */
631	RTCCUINTREG uCr4 = ASMGetCR4();
632	if (!(uCr4 & X86_CR4_VMXE))
633	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
634
635	/* Enter VMX root mode. */
636	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
637	if (RT_FAILURE(rc))
638	ASMSetCR4(uCr4);
639
640	/* Restore interrupts. */
641	ASMSetFlags(fFlags);
642	return rc;
643	}
644
645
646	/**
647	* Exits VMX root mode operation on the current CPU.
648	*
649	* @returns VBox status code.
650	*/
651	static int hmR0VmxLeaveRootMode(void)
652	{
653	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
654	RTCCUINTREG fFlags = ASMIntDisableFlags();
655	int rc = VINF_SUCCESS;
656
657	/* If we're for some reason not in VMX root mode, then don't leave it. */
658	if (ASMGetCR4() & X86_CR4_VMXE)
659	{
660	/* Exit VMX root mode and clear the VMX bit in CR4 */
661	VMXDisable();
662	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
663	}
664	else
665	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
666
667	/* Restore interrupts. */
668	ASMSetFlags(fFlags);
669	return rc;
670	}
671
672
673	/**
674	* Allocates and maps one physically contiguous page. The allocated page is
675	* zero'd out. (Used by various VT-x structures).
676	*
677	* @returns IPRT status code.
678	* @param pMemObj Pointer to the ring-0 memory object.
679	* @param ppVirt Where to store the virtual address of the
680	* allocation.
681	* @param pPhys Where to store the physical address of the
682	* allocation.
683	*/
684	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
685	{
686	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
687	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
688	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
689
690	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
691	if (RT_FAILURE(rc))
692	return rc;
693	ppVirt = RTR0MemObjAddress(pMemObj);
694	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
695	ASMMemZero32(*ppVirt, PAGE_SIZE);
696	return VINF_SUCCESS;
697	}
698
699
700	/**
701	* Frees and unmaps an allocated physical page.
702	*
703	* @param pMemObj Pointer to the ring-0 memory object.
704	* @param ppVirt Where to re-initialize the virtual address of
705	* allocation as 0.
706	* @param pHCPhys Where to re-initialize the physical address of the
707	* allocation as 0.
708	*/
709	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
710	{
711	AssertPtr(pMemObj);
712	AssertPtr(ppVirt);
713	AssertPtr(pHCPhys);
714	if (*pMemObj != NIL_RTR0MEMOBJ)
715	{
716	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
717	AssertRC(rc);
718	*pMemObj = NIL_RTR0MEMOBJ;
719	*ppVirt = 0;
720	*pHCPhys = 0;
721	}
722	}
723
724
725	/**
726	* Worker function to free VT-x related structures.
727	*
728	* @returns IPRT status code.
729	* @param pVM Pointer to the VM.
730	*/
731	static void hmR0VmxStructsFree(PVM pVM)
732	{
733	for (VMCPUID i = 0; i < pVM->cCpus; i++)
734	{
735	PVMCPU pVCpu = &pVM->aCpus[i];
736	AssertPtr(pVCpu);
737
738	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
739	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
740
741	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
742	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
743
744	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
745	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
746	}
747
748	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
749	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
750	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
751	#endif
752	}
753
754
755	/**
756	* Worker function to allocate VT-x related VM structures.
757	*
758	* @returns IPRT status code.
759	* @param pVM Pointer to the VM.
760	*/
761	static int hmR0VmxStructsAlloc(PVM pVM)
762	{
763	/*
764	* Initialize members up-front so we can cleanup properly on allocation failure.
765	*/
766	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
767	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
768	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
769	pVM->hm.s.vmx.HCPhys##a_Name = 0;
770
771	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
772	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
773	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
774	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
775
776	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
777	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
778	#endif
779	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
780
781	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
782	for (VMCPUID i = 0; i < pVM->cCpus; i++)
783	{
784	PVMCPU pVCpu = &pVM->aCpus[i];
785	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
786	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
787	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
788	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
789	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
790	}
791	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
792	#undef VMXLOCAL_INIT_VM_MEMOBJ
793
794	/*
795	* Allocate all the VT-x structures.
796	*/
797	int rc = VINF_SUCCESS;
798	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
799	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
800	if (RT_FAILURE(rc))
801	goto cleanup;
802	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
803	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
804	#endif
805
806	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
807	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
808	{
809	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
810	&pVM->hm.s.vmx.HCPhysApicAccess);
811	if (RT_FAILURE(rc))
812	goto cleanup;
813	}
814
815	/*
816	* Initialize per-VCPU VT-x structures.
817	*/
818	for (VMCPUID i =0; i < pVM->cCpus; i++)
819	{
820	PVMCPU pVCpu = &pVM->aCpus[i];
821	AssertPtr(pVCpu);
822
823	/* Allocate the VM control structure (VMCS). */
824	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
825	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
826	if (RT_FAILURE(rc))
827	goto cleanup;
828
829	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
830	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
831	{
832	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
833	&pVCpu->hm.s.vmx.HCPhysVirtApic);
834	if (RT_FAILURE(rc))
835	goto cleanup;
836	}
837
838	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
839	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
840	{
841	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
842	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
843	if (RT_FAILURE(rc))
844	goto cleanup;
845	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
846	}
847
848	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
849	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
850	if (RT_FAILURE(rc))
851	goto cleanup;
852
853	/* Allocate the VM-exit MSR-load page for the host MSRs. */
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
855	if (RT_FAILURE(rc))
856	goto cleanup;
857	}
858
859	return VINF_SUCCESS;
860
861	cleanup:
862	hmR0VmxStructsFree(pVM);
863	return rc;
864	}
865
866
867	/**
868	* Does global VT-x initialization (called during module initialization).
869	*
870	* @returns VBox status code.
871	*/
872	VMMR0DECL(int) VMXR0GlobalInit(void)
873	{
874	#ifdef HMVMX_USE_FUNCTION_TABLE
875	/* Setup the main VM exit handlers. */
876	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
877	# ifdef VBOX_STRICT
878	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
879	Assert(g_apfnVMExitHandlers[i]);
880	# endif
881	#endif
882	return VINF_SUCCESS;
883	}
884
885
886	/**
887	* Does global VT-x termination (called during module termination).
888	*/
889	VMMR0DECL(void) VMXR0GlobalTerm()
890	{
891	/* Nothing to do currently. */
892	}
893
894
895	/**
896	* Sets up and activates VT-x on the current CPU.
897	*
898	* @returns VBox status code.
899	* @param pCpu Pointer to the global CPU info struct.
900	* @param pVM Pointer to the VM (can be NULL after a host resume
901	* operation).
902	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
903	* fEnabledByHost is true).
904	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
905	* @a fEnabledByHost is true).
906	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
907	* enable VT-x/AMD-V on the host.
908	*/
909	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
910	{
911	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
912	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
913
914	if (!fEnabledByHost)
915	{
916	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
917	if (RT_FAILURE(rc))
918	return rc;
919	}
920
921	/*
922	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
923	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
924	* each time while reusing a VPID after hitting the MaxASID limit once.
925	*/
926	if ( pVM
927	&& pVM->hm.s.vmx.fVpid
928	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
929	{
930	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
931	pCpu->fFlushAsidBeforeUse = false;
932	}
933	else
934	pCpu->fFlushAsidBeforeUse = true;
935
936	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
937	++pCpu->cTlbFlushes;
938
939	return VINF_SUCCESS;
940	}
941
942
943	/**
944	* Deactivates VT-x on the current CPU.
945	*
946	* @returns VBox status code.
947	* @param pCpu Pointer to the global CPU info struct.
948	* @param pvCpuPage Pointer to the VMXON region.
949	* @param HCPhysCpuPage Physical address of the VMXON region.
950	*/
951	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
952	{
953	NOREF(pCpu);
954	NOREF(pvCpuPage);
955	NOREF(HCPhysCpuPage);
956
957	hmR0VmxLeaveRootMode();
958	return VINF_SUCCESS;
959	}
960
961
962	/**
963	* Sets the permission bits for the specified MSR in the MSR bitmap.
964	*
965	* @param pVCpu Pointer to the VMCPU.
966	* @param uMSR The MSR value.
967	* @param enmRead Whether reading this MSR causes a VM-exit.
968	* @param enmWrite Whether writing this MSR causes a VM-exit.
969	*/
970	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
971	{
972	int32_t iBit;
973	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
974
975	/*
976	* Layout:
977	* 0x000 - 0x3ff - Low MSR read bits
978	* 0x400 - 0x7ff - High MSR read bits
979	* 0x800 - 0xbff - Low MSR write bits
980	* 0xc00 - 0xfff - High MSR write bits
981	*/
982	if (uMsr <= 0x00001FFF)
983	{
984	/* Pentium-compatible MSRs */
985	iBit = uMsr;
986	}
987	else if ( uMsr >= 0xC0000000
988	&& uMsr <= 0xC0001FFF)
989	{
990	/* AMD Sixth Generation x86 Processor MSRs */
991	iBit = (uMsr - 0xC0000000);
992	pbMsrBitmap += 0x400;
993	}
994	else
995	{
996	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
997	return;
998	}
999
1000	Assert(iBit <= 0x1fff);
1001	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1002	ASMBitSet(pbMsrBitmap, iBit);
1003	else
1004	ASMBitClear(pbMsrBitmap, iBit);
1005
1006	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1007	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1008	else
1009	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1010	}
1011
1012
1013	/**
1014	* Flushes the TLB using EPT.
1015	*
1016	* @returns VBox status code.
1017	* @param pVM Pointer to the VM.
1018	* @param pVCpu Pointer to the VMCPU.
1019	* @param enmFlush Type of flush.
1020	*/
1021	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1022	{
1023	AssertPtr(pVM);
1024	Assert(pVM->hm.s.fNestedPaging);
1025
1026	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1027
1028	uint64_t descriptor[2];
1029	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1030	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1031
1032	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1033	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1034	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1035	}
1036
1037
1038	/**
1039	* Flushes the TLB using VPID.
1040	*
1041	* @returns VBox status code.
1042	* @param pVM Pointer to the VM.
1043	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1044	* enmFlush).
1045	* @param enmFlush Type of flush.
1046	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1047	* on @a enmFlush).
1048	*/
1049	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1050	{
1051	AssertPtr(pVM);
1052	Assert(pVM->hm.s.vmx.fVpid);
1053
1054	uint64_t descriptor[2];
1055	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1056	{
1057	descriptor[0] = 0;
1058	descriptor[1] = 0;
1059	}
1060	else
1061	{
1062	AssertPtr(pVCpu);
1063	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1064	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1065	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1066	descriptor[1] = GCPtr;
1067	}
1068
1069	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1070	AssertMsg(rc == VINF_SUCCESS,
1071	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1072	if ( RT_SUCCESS(rc)
1073	&& pVCpu)
1074	{
1075	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1076	}
1077	}
1078
1079
1080	/**
1081	* Invalidates a guest page by guest virtual address. Only relevant for
1082	* EPT/VPID, otherwise there is nothing really to invalidate.
1083	*
1084	* @returns VBox status code.
1085	* @param pVM Pointer to the VM.
1086	* @param pVCpu Pointer to the VMCPU.
1087	* @param GCVirt Guest virtual address of the page to invalidate.
1088	*/
1089	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1090	{
1091	AssertPtr(pVM);
1092	AssertPtr(pVCpu);
1093	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1094
1095	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1096	if (!fFlushPending)
1097	{
1098	/*
1099	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1100	* See @bugref{6043} and @bugref{6177}.
1101	*
1102	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1103	* function maybe called in a loop with individual addresses.
1104	*/
1105	if (pVM->hm.s.vmx.fVpid)
1106	{
1107	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1108	{
1109	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1110	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1111	}
1112	else
1113	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	}
1115	else if (pVM->hm.s.fNestedPaging)
1116	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1117	}
1118
1119	return VINF_SUCCESS;
1120	}
1121
1122
1123	/**
1124	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1125	* otherwise there is nothing really to invalidate.
1126	*
1127	* @returns VBox status code.
1128	* @param pVM Pointer to the VM.
1129	* @param pVCpu Pointer to the VMCPU.
1130	* @param GCPhys Guest physical address of the page to invalidate.
1131	*/
1132	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1133	{
1134	LogFlowFunc(("%RGp\n", GCPhys));
1135
1136	/*
1137	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1138	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1139	* This function might be called in a loop.
1140	*/
1141	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1143	return VINF_SUCCESS;
1144	}
1145
1146
1147	/**
1148	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1149	* case where neither EPT nor VPID is supported by the CPU.
1150	*
1151	* @param pVM Pointer to the VM.
1152	* @param pVCpu Pointer to the VMCPU.
1153	*
1154	* @remarks Called with interrupts disabled.
1155	*/
1156	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1157	{
1158	NOREF(pVM);
1159	AssertPtr(pVCpu);
1160	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1161	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1162
1163	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1164	AssertPtr(pCpu);
1165
1166	pVCpu->hm.s.TlbShootdown.cPages = 0;
1167	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1168	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1169	pVCpu->hm.s.fForceTLBFlush = false;
1170	return;
1171	}
1172
1173
1174	/**
1175	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1176	*
1177	* @param pVM Pointer to the VM.
1178	* @param pVCpu Pointer to the VMCPU.
1179	* @remarks All references to "ASID" in this function pertains to "VPID" in
1180	* Intel's nomenclature. The reason is, to avoid confusion in compare
1181	* statements since the host-CPU copies are named "ASID".
1182	*
1183	* @remarks Called with interrupts disabled.
1184	*/
1185	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1186	{
1187	AssertPtr(pVM);
1188	AssertPtr(pVCpu);
1189	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1190	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1191	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1192
1193	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1194	AssertPtr(pCpu);
1195
1196	/*
1197	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1198	* This can happen both for start & resume due to long jumps back to ring-3.
1199	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1200	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1201	*/
1202	bool fNewASID = false;
1203	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1204	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1205	{
1206	pVCpu->hm.s.fForceTLBFlush = true;
1207	fNewASID = true;
1208	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1209	}
1210
1211	/*
1212	* Check for explicit TLB shootdowns.
1213	*/
1214	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1215	{
1216	pVCpu->hm.s.fForceTLBFlush = true;
1217	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1218	}
1219
1220	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1221	if (pVCpu->hm.s.fForceTLBFlush)
1222	{
1223	if (fNewASID)
1224	{
1225	++pCpu->uCurrentAsid;
1226	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1227	{
1228	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1229	pCpu->cTlbFlushes++;
1230	pCpu->fFlushAsidBeforeUse = true;
1231	}
1232
1233	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1234	if (pCpu->fFlushAsidBeforeUse)
1235	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1236	}
1237	else
1238	{
1239	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1240	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1241	else
1242	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1243	}
1244
1245	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1246	pVCpu->hm.s.fForceTLBFlush = false;
1247	}
1248	else
1249	{
1250	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1251	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1252	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1253	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1254
1255	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1256	* not be executed. See hmQueueInvlPage() where it is commented
1257	* out. Support individual entry flushing someday. */
1258	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1259	{
1260	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1261
1262	/*
1263	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1264	* as supported by the CPU.
1265	*/
1266	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1267	{
1268	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1269	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1270	}
1271	else
1272	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1273	}
1274	else
1275	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1276	}
1277	pVCpu->hm.s.TlbShootdown.cPages = 0;
1278	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1279
1280	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1281	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1282	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1283	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1284	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1285	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1286
1287	/* Update VMCS with the VPID. */
1288	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1289	AssertRC(rc);
1290	}
1291
1292
1293	/**
1294	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1295	*
1296	* @returns VBox status code.
1297	* @param pVM Pointer to the VM.
1298	* @param pVCpu Pointer to the VMCPU.
1299	*
1300	* @remarks Called with interrupts disabled.
1301	*/
1302	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1303	{
1304	AssertPtr(pVM);
1305	AssertPtr(pVCpu);
1306	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1307	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1308
1309	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1310	AssertPtr(pCpu);
1311
1312	/*
1313	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1314	* This can happen both for start & resume due to long jumps back to ring-3.
1315	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1316	*/
1317	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1318	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1319	{
1320	pVCpu->hm.s.fForceTLBFlush = true;
1321	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1322	}
1323
1324	/* Check for explicit TLB shootdown flushes. */
1325	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1326	{
1327	pVCpu->hm.s.fForceTLBFlush = true;
1328	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1329	}
1330
1331	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1332	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1333
1334	if (pVCpu->hm.s.fForceTLBFlush)
1335	{
1336	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1337	pVCpu->hm.s.fForceTLBFlush = false;
1338	}
1339	else
1340	{
1341	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1342	* not be executed. See hmQueueInvlPage() where it is commented
1343	* out. Support individual entry flushing someday. */
1344	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1345	{
1346	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1347	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1348	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1349	}
1350	else
1351	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1352	}
1353
1354	pVCpu->hm.s.TlbShootdown.cPages = 0;
1355	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1356	}
1357
1358
1359	/**
1360	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1361	*
1362	* @returns VBox status code.
1363	* @param pVM Pointer to the VM.
1364	* @param pVCpu Pointer to the VMCPU.
1365	*
1366	* @remarks Called with interrupts disabled.
1367	*/
1368	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1369	{
1370	AssertPtr(pVM);
1371	AssertPtr(pVCpu);
1372	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1373	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1374
1375	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1376
1377	/*
1378	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1379	* This can happen both for start & resume due to long jumps back to ring-3.
1380	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1381	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1382	*/
1383	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1384	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1385	{
1386	pVCpu->hm.s.fForceTLBFlush = true;
1387	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1388	}
1389
1390	/* Check for explicit TLB shootdown flushes. */
1391	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1392	{
1393	/*
1394	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1395	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1396	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1397	*/
1398	pVCpu->hm.s.fForceTLBFlush = true;
1399	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1400	}
1401
1402	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1403	if (pVCpu->hm.s.fForceTLBFlush)
1404	{
1405	++pCpu->uCurrentAsid;
1406	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1407	{
1408	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1409	pCpu->fFlushAsidBeforeUse = true;
1410	pCpu->cTlbFlushes++;
1411	}
1412
1413	pVCpu->hm.s.fForceTLBFlush = false;
1414	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1415	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1416	if (pCpu->fFlushAsidBeforeUse)
1417	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1418	}
1419	else
1420	{
1421	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1422	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1423	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1424	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1425
1426	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1427	* not be executed. See hmQueueInvlPage() where it is commented
1428	* out. Support individual entry flushing someday. */
1429	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1430	{
1431	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1432	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1433	{
1434	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1435	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1436	}
1437	else
1438	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1439	}
1440	else
1441	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1442	}
1443
1444	pVCpu->hm.s.TlbShootdown.cPages = 0;
1445	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1446
1447	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1448	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1449	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1450	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1451	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1452	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1453
1454	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1455	AssertRC(rc);
1456	}
1457
1458
1459	/**
1460	* Flushes the guest TLB entry based on CPU capabilities.
1461	*
1462	* @param pVCpu Pointer to the VMCPU.
1463	*/
1464	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1465	{
1466	PVM pVM = pVCpu->CTX_SUFF(pVM);
1467	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1468	{
1469	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1470	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1471	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1472	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1473	default:
1474	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1475	break;
1476	}
1477	}
1478
1479
1480	/**
1481	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1482	* TLB entries from the host TLB before VM-entry.
1483	*
1484	* @returns VBox status code.
1485	* @param pVM Pointer to the VM.
1486	*/
1487	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1488	{
1489	/*
1490	* Determine optimal flush type for nested paging.
1491	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1492	* guest execution (see hmR3InitFinalizeR0()).
1493	*/
1494	if (pVM->hm.s.fNestedPaging)
1495	{
1496	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1497	{
1498	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1499	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1500	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1501	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1502	else
1503	{
1504	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1505	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1506	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1507	}
1508
1509	/* Make sure the write-back cacheable memory type for EPT is supported. */
1510	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1511	{
1512	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1513	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1514	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1515	}
1516	}
1517	else
1518	{
1519	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1520	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1521	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1522	}
1523	}
1524
1525	/*
1526	* Determine optimal flush type for VPID.
1527	*/
1528	if (pVM->hm.s.vmx.fVpid)
1529	{
1530	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1531	{
1532	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1533	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1534	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1535	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1536	else
1537	{
1538	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1539	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1540	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1541	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1542	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1543	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1544	pVM->hm.s.vmx.fVpid = false;
1545	}
1546	}
1547	else
1548	{
1549	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1550	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1552	pVM->hm.s.vmx.fVpid = false;
1553	}
1554	}
1555
1556	/*
1557	* Setup the handler for flushing tagged-TLBs.
1558	*/
1559	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1560	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1561	else if (pVM->hm.s.fNestedPaging)
1562	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1563	else if (pVM->hm.s.vmx.fVpid)
1564	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1565	else
1566	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1567	return VINF_SUCCESS;
1568	}
1569
1570
1571	/**
1572	* Sets up pin-based VM-execution controls in the VMCS.
1573	*
1574	* @returns VBox status code.
1575	* @param pVM Pointer to the VM.
1576	* @param pVCpu Pointer to the VMCPU.
1577	*/
1578	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1579	{
1580	AssertPtr(pVM);
1581	AssertPtr(pVCpu);
1582
1583	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1584	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1585
1586	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1587	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1588	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1589
1590	/* Enable the VMX preemption timer. */
1591	if (pVM->hm.s.vmx.fUsePreemptTimer)
1592	{
1593	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1594	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1595	}
1596
1597	if ((val & zap) != val)
1598	{
1599	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1600	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1601	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1602	}
1603
1604	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1605	AssertRCReturn(rc, rc);
1606
1607	/* Update VCPU with the currently set pin-based VM-execution controls. */
1608	pVCpu->hm.s.vmx.u32PinCtls = val;
1609	return rc;
1610	}
1611
1612
1613	/**
1614	* Sets up processor-based VM-execution controls in the VMCS.
1615	*
1616	* @returns VBox status code.
1617	* @param pVM Pointer to the VM.
1618	* @param pVMCPU Pointer to the VMCPU.
1619	*/
1620	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1621	{
1622	AssertPtr(pVM);
1623	AssertPtr(pVCpu);
1624
1625	int rc = VERR_INTERNAL_ERROR_5;
1626	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1627	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1628
1629	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1630	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1632	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1633	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1634	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1635	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1636
1637	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1638	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1639	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1640	{
1641	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1642	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1643	}
1644
1645	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1646	if (!pVM->hm.s.fNestedPaging)
1647	{
1648	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1651	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1652	}
1653
1654	/* Use TPR shadowing if supported by the CPU. */
1655	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1656	{
1657	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1658	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1659	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1660	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1661	AssertRCReturn(rc, rc);
1662
1663	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1664	/* CR8 writes causes a VM-exit based on TPR threshold. */
1665	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1666	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1667	}
1668	else
1669	{
1670	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1671	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1672	}
1673
1674	/* Use MSR-bitmaps if supported by the CPU. */
1675	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1676	{
1677	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1678
1679	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1680	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1681	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1682	AssertRCReturn(rc, rc);
1683
1684	/*
1685	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1686	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1687	*/
1688	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1689	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1690	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1691	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1692	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1693	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1694	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1695	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1696	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1697	}
1698
1699	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1700	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1701	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1702
1703	if ((val & zap) != val)
1704	{
1705	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1706	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1707	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1708	}
1709
1710	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1711	AssertRCReturn(rc, rc);
1712
1713	/* Update VCPU with the currently set processor-based VM-execution controls. */
1714	pVCpu->hm.s.vmx.u32ProcCtls = val;
1715
1716	/*
1717	* Secondary processor-based VM-execution controls.
1718	*/
1719	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1720	{
1721	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1722	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1723
1724	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1725	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1726
1727	if (pVM->hm.s.fNestedPaging)
1728	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1729	else
1730	{
1731	/*
1732	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1733	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1734	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1735	*/
1736	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1737	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1738	}
1739
1740	if (pVM->hm.s.vmx.fVpid)
1741	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1742
1743	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1744	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1745
1746	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1747	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1748	* done dynamically. */
1749	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1750	{
1751	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1752	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1753	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1754	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1755	AssertRCReturn(rc, rc);
1756	}
1757
1758	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1759	{
1760	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1762	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1763	}
1764
1765	if ((val & zap) != val)
1766	{
1767	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1768	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1769	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1770	}
1771
1772	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1773	AssertRCReturn(rc, rc);
1774
1775	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1776	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1777	}
1778
1779	return VINF_SUCCESS;
1780	}
1781
1782
1783	/**
1784	* Sets up miscellaneous (everything other than Pin & Processor-based
1785	* VM-execution) control fields in the VMCS.
1786	*
1787	* @returns VBox status code.
1788	* @param pVM Pointer to the VM.
1789	* @param pVCpu Pointer to the VMCPU.
1790	*/
1791	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1792	{
1793	AssertPtr(pVM);
1794	AssertPtr(pVCpu);
1795
1796	int rc = VERR_GENERAL_FAILURE;
1797
1798	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1799	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1800
1801	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1802
1803	/*
1804	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1805	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1806	* We thus use the exception bitmap to control it rather than use both.
1807	*/
1808	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1809	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1810
1811	/** @todo Explore possibility of using IO-bitmaps. */
1812	/* All IO & IOIO instructions cause VM-exits. */
1813	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1814	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1815
1816	/* Setup MSR autoloading/autostoring. */
1817	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1818	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1819	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1820	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1821	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1822	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1823
1824	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1825	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1826	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1827	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1828
1829	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1830	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1831
1832	/* Setup debug controls */
1833	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1834	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1835	AssertRCReturn(rc, rc);
1836	return rc;
1837	}
1838
1839
1840	/**
1841	* Sets up the initial exception bitmap in the VMCS based on static conditions
1842	* (i.e. conditions that cannot ever change at runtime).
1843	*
1844	* @returns VBox status code.
1845	* @param pVM Pointer to the VM.
1846	* @param pVCpu Pointer to the VMCPU.
1847	*/
1848	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1849	{
1850	AssertPtr(pVM);
1851	AssertPtr(pVCpu);
1852
1853	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1854
1855	uint32_t u32XcptBitmap = 0;
1856
1857	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1858	if (!pVM->hm.s.fNestedPaging)
1859	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1860
1861	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1862	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1863	AssertRCReturn(rc, rc);
1864	return rc;
1865	}
1866
1867
1868	/**
1869	* Sets up the initial guest-state mask. The guest-state mask is consulted
1870	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1871	* for the nested virtualization case (as it would cause a VM-exit).
1872	*
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1876	{
1877	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1878	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1879	return VINF_SUCCESS;
1880	}
1881
1882
1883	/**
1884	* Does per-VM VT-x initialization.
1885	*
1886	* @returns VBox status code.
1887	* @param pVM Pointer to the VM.
1888	*/
1889	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1890	{
1891	LogFlowFunc(("pVM=%p\n", pVM));
1892
1893	int rc = hmR0VmxStructsAlloc(pVM);
1894	if (RT_FAILURE(rc))
1895	{
1896	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1897	return rc;
1898	}
1899
1900	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1901	{
1902	PVMCPU pVCpu = &pVM->aCpus[i];
1903
1904	/* Current guest paging mode. */
1905	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1906	}
1907
1908	return VINF_SUCCESS;
1909	}
1910
1911
1912	/**
1913	* Does per-VM VT-x termination.
1914	*
1915	* @returns VBox status code.
1916	* @param pVM Pointer to the VM.
1917	*/
1918	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1919	{
1920	LogFlowFunc(("pVM=%p\n", pVM));
1921
1922	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1923	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1924	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1925	#endif
1926	hmR0VmxStructsFree(pVM);
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Sets up the VM for execution under VT-x.
1933	* This function is only called once per-VM during initalization.
1934	*
1935	* @returns VBox status code.
1936	* @param pVM Pointer to the VM.
1937	*/
1938	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1939	{
1940	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1941	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1942
1943	LogFlowFunc(("pVM=%p\n", pVM));
1944
1945	/*
1946	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1947	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1948	*/
1949	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1950	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1951	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1952	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1953	{
1954	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1955	return VERR_INTERNAL_ERROR;
1956	}
1957
1958	/* Initialize these always, see hmR3InitFinalizeR0().*/
1959	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1960	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1961
1962	/* Setup the tagged-TLB flush handlers. */
1963	int rc = hmR0VmxSetupTaggedTlb(pVM);
1964	if (RT_FAILURE(rc))
1965	{
1966	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1967	return rc;
1968	}
1969
1970	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1971	{
1972	PVMCPU pVCpu = &pVM->aCpus[i];
1973	AssertPtr(pVCpu);
1974	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1975
1976	/* Set revision dword at the beginning of the VMCS structure. */
1977	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1978
1979	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1980	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1981	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1982	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1983
1984	/* Load this VMCS as the current VMCS. */
1985	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1986	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1987	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1988
1989	/* Setup the pin-based VM-execution controls. */
1990	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1991	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1992	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1993
1994	/* Setup the processor-based VM-execution controls. */
1995	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1996	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1997	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1998
1999	/* Setup the rest (miscellaneous) VM-execution controls. */
2000	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2001	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2002	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2003
2004	/* Setup the initial exception bitmap. */
2005	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2006	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2007	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2008
2009	/* Setup the initial guest-state mask. */
2010	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2011	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2012	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2013
2014	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2015	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
2016	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019	#endif
2020
2021	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2022	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2023	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2024	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2025
2026	/* Update the last error record for this VCPU. */
2027	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2028	}
2029
2030	return VINF_SUCCESS;
2031	}
2032
2033
2034	/**
2035	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2036	* the VMCS.
2037	*
2038	* @returns VBox status code.
2039	* @param pVM Pointer to the VM.
2040	* @param pVCpu Pointer to the VMCPU.
2041	*/
2042	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2043	{
2044	RTCCUINTREG uReg = ASMGetCR0();
2045	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2046
2047	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2048	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2049	if (HMVMX_IS_64BIT_HOST_MODE())
2050	{
2051	uint64_t uReg = hmR0Get64bitCR3();
2052	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2053	}
2054	else
2055	#endif
2056	{
2057	uReg = ASMGetCR3();
2058	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2059	}
2060
2061	uReg = ASMGetCR4();
2062	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2063	AssertRCReturn(rc, rc);
2064	return rc;
2065	}
2066
2067
2068	/**
2069	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2070	* the host-state area in the VMCS.
2071	*
2072	* @returns VBox status code.
2073	* @param pVM Pointer to the VM.
2074	* @param pVCpu Pointer to the VMCPU.
2075	*/
2076	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2077	{
2078	int rc = VERR_INTERNAL_ERROR_5;
2079	RTSEL uSelCS = 0;
2080	RTSEL uSelSS = 0;
2081	RTSEL uSelDS = 0;
2082	RTSEL uSelES = 0;
2083	RTSEL uSelFS = 0;
2084	RTSEL uSelGS = 0;
2085	RTSEL uSelTR = 0;
2086
2087	/*
2088	* Host Selector registers.
2089	*/
2090	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2091	if (HMVMX_IS_64BIT_HOST_MODE())
2092	{
2093	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2094	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2095	}
2096	else
2097	{
2098	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2099	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2100	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2101	}
2102	#else
2103	uSelCS = ASMGetCS();
2104	uSelSS = ASMGetSS();
2105	#endif
2106
2107	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2108	/** @todo Verify if we have any platform that actually run with DS or ES with
2109	* RPL != 0 in kernel space. */
2110	uSelDS = 0;
2111	uSelES = 0;
2112	uSelFS = 0;
2113	uSelGS = 0;
2114	uSelTR = ASMGetTR();
2115
2116	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2117	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2118	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2119	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2120	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2121	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2122	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2123	Assert(uSelCS != 0);
2124	Assert(uSelTR != 0);
2125
2126	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2127	#if 0
2128	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2129	Assert(uSelSS != 0);
2130	#endif
2131
2132	/* Write these host selector fields into the host-state area in the VMCS. */
2133	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2134	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2135	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2136	#if 0
2137	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2138	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2139	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2140	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2141	#endif
2142	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2143	AssertRCReturn(rc, rc);
2144
2145	/*
2146	* Host GDTR and IDTR.
2147	*/
2148	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2149	* be safe to -not- save and restore GDTR and IDTR in the assembly
2150	* code and just do it here and don't care if the limits are zapped on
2151	* VM-exit. */
2152	RTGDTR Gdtr;
2153	RT_ZERO(Gdtr);
2154	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2155	if (HMVMX_IS_64BIT_HOST_MODE())
2156	{
2157	X86XDTR64 Gtr64;
2158	X86XDTR64 Idtr64;
2159	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2160	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2161	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2162	Gdtr.cbGdt = Gdtr64.cb;
2163	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2164	}
2165	else
2166	#endif
2167	{
2168	RTIDTR Idtr;
2169	ASMGetGDTR(&Gdtr);
2170	ASMGetIDTR(&Idtr);
2171	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2172	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2173	}
2174	AssertRCReturn(rc, rc);
2175
2176	/*
2177	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2178	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2179	*/
2180	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2181	{
2182	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2183	return VERR_VMX_INVALID_HOST_STATE;
2184	}
2185
2186	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2187	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2188	if (HMVMX_IS_64BIT_HOST_MODE())
2189	{
2190	/* We need the 64-bit TR base for hybrid darwin. */
2191	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2192	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2193	}
2194	else
2195	#endif
2196	{
2197	uintptr_t uTRBase;
2198	#if HC_ARCH_BITS == 64
2199	uTRBase = X86DESC64_BASE(pDesc);
2200	#else
2201	uTRBase = X86DESC_BASE(pDesc);
2202	#endif
2203	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2204	}
2205	AssertRCReturn(rc, rc);
2206
2207	/*
2208	* Host FS base and GS base.
2209	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2210	* would take care of the bases. In 64-bit, the MSRs come into play.
2211	*/
2212	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2213	if (HMVMX_IS_64BIT_HOST_MODE())
2214	{
2215	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2216	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2217	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2218	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2219	AssertRCReturn(rc, rc);
2220	}
2221	#endif
2222	return rc;
2223	}
2224
2225
2226	/**
2227	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2228	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2229	* the host after every successful VM exit.
2230	*
2231	* @returns VBox status code.
2232	* @param pVM Pointer to the VM.
2233	* @param pVCpu Pointer to the VMCPU.
2234	*/
2235	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2236	{
2237	AssertPtr(pVCpu);
2238	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2239
2240	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2241	uint32_t cHostMsrs = 0;
2242	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2243
2244	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2245	{
2246	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2247	pHostMsr->u32Reserved = 0;
2248	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2249	if (CPUMIsGuestInLongMode(pVCpu))
2250	{
2251	/* Must match the EFER value in our 64 bits switcher. */
2252	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2253	}
2254	else
2255	#endif
2256	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2257	pHostMsr++; cHostMsrs++;
2258	}
2259
2260	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2261	if (HMVMX_IS_64BIT_HOST_MODE())
2262	{
2263	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2264	pHostMsr->u32Reserved = 0;
2265	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2266	pHostMsr++; cHostMsrs++;
2267	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2268	pHostMsr->u32Reserved = 0;
2269	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2270	pHostMsr++; cHostMsrs++;
2271	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2272	pHostMsr->u32Reserved = 0;
2273	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2274	pHostMsr++; cHostMsrs++;
2275	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2276	#if 0
2277	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2278	pMsr->u32Reserved = 0;
2279	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2280	pHostMsr++; cHostMsrs++;
2281	#endif
2282	}
2283	#endif
2284
2285	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2286	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2287	{
2288	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2289	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2290	}
2291
2292	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2293
2294	/*
2295	* Host Sysenter MSRs.
2296	*/
2297	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2298	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2299	if (HMVMX_IS_64BIT_HOST_MODE())
2300	{
2301	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2302	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2303	}
2304	else
2305	{
2306	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2308	}
2309	#elif HC_ARCH_BITS == 32
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2311	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2312	#else
2313	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2314	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2315	#endif
2316	AssertRCReturn(rc, rc);
2317
2318	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2319	* hmR0VmxSetupExitCtls() !! */
2320	return rc;
2321	}
2322
2323
2324	/**
2325	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2326	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2327	* controls".
2328	*
2329	* @returns VBox status code.
2330	* @param pVM Pointer to the VM.
2331	* @param pVCpu Pointer to the VMCPU.
2332	* @param pCtx Pointer to the guest-CPU context.
2333	*
2334	* @remarks No-long-jump zone!!!
2335	*/
2336	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2337	{
2338	int rc = VINF_SUCCESS;
2339	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2340	{
2341	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2342	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2343
2344	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2345	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2346
2347	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2348	if (CPUMIsGuestInLongModeEx(pCtx))
2349	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2350	else
2351	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2352
2353	/*
2354	* The following should not be set (since we're not in SMM mode):
2355	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2356	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2357	*/
2358
2359	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2360	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2361	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2362
2363	if ((val & zap) != val)
2364	{
2365	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2366	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2367	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2368	}
2369
2370	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2371	AssertRCReturn(rc, rc);
2372
2373	/* Update VCPU with the currently set VM-exit controls. */
2374	pVCpu->hm.s.vmx.u32EntryCtls = val;
2375	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2376	}
2377	return rc;
2378	}
2379
2380
2381	/**
2382	* Sets up the VM-exit controls in the VMCS.
2383	*
2384	* @returns VBox status code.
2385	* @param pVM Pointer to the VM.
2386	* @param pVCpu Pointer to the VMCPU.
2387	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2388	* out-of-sync. Make sure to update the required fields
2389	* before using them.
2390	*
2391	* @remarks requires EFER.
2392	*/
2393	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2394	{
2395	int rc = VINF_SUCCESS;
2396	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2397	{
2398	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2399	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2400
2401	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2402	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2403
2404	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2405	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2406	if (HMVMX_IS_64BIT_HOST_MODE())
2407	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2408	else
2409	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2410	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2411	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2412	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2413	else
2414	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2415	#endif
2416
2417	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2418	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2419
2420	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2421	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2422	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2423	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2424	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2425
2426	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2427	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2428
2429	if ((val & zap) != val)
2430	{
2431	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2432	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2434	}
2435
2436	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2437	AssertRCReturn(rc, rc);
2438
2439	/* Update VCPU with the currently set VM-exit controls. */
2440	pVCpu->hm.s.vmx.u32ExitCtls = val;
2441	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2442	}
2443	return rc;
2444	}
2445
2446
2447	/**
2448	* Loads the guest APIC and related state.
2449	*
2450	* @returns VBox status code.
2451	* @param pVM Pointer to the VM.
2452	* @param pVCpu Pointer to the VMCPU.
2453	* @param pCtx Pointer to the guest-CPU context.
2454	*/
2455	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2456	{
2457	int rc = VINF_SUCCESS;
2458	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2459	{
2460	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2461	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2462	{
2463	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2464
2465	bool fPendingIntr = false;
2466	uint8_t u8GuestTpr = 0;
2467	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2468	AssertRCReturn(rc, rc);
2469
2470	/*
2471	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2472	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2473	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2474	* the interrupt when we VM-exit for other reasons.
2475	*/
2476	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2477	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2478	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2479	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2480
2481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2482	AssertRCReturn(rc, rc);
2483
2484	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2485	if (pVM->hm.s.fTPRPatchingActive)
2486	{
2487	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2488	pCtx->msrLSTAR = u8GuestTpr;
2489	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2490	{
2491	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2492	if (fPendingIntr)
2493	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2494	else
2495	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496	}
2497	}
2498	}
2499
2500	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2501	}
2502	return rc;
2503	}
2504
2505
2506	/**
2507	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2508	*
2509	* @returns
2510	* @param pVCpu Pointer to the VMCPU.
2511	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2512	* out-of-sync. Make sure to update the required fields
2513	* before using them.
2514	*
2515	* @remarks No-long-jump zone!!!
2516	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2517	*/
2518	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2519	{
2520	/*
2521	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2522	* inhibit interrupts or clear any existing interrupt-inhibition.
2523	*/
2524	uint32_t uIntrState = 0;
2525	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2526	{
2527	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2528	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2529	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2530	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2531	{
2532	/*
2533	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2534	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2535	*/
2536	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2537	}
2538	else if (pMixedCtx->eflags.Bits.u1IF)
2539	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2540	else
2541	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2542	}
2543	return uIntrState;
2544	}
2545
2546
2547	/**
2548	* Loads the guest's interruptibility-state into the guest-state area in the
2549	* VMCS.
2550	*
2551	* @returns VBox status code.
2552	* @param pVCpu Pointer to the VMCPU.
2553	* @param uIntrState The interruptibility-state to set.
2554	*/
2555	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2556	{
2557	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2558	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2559	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2560	AssertRCReturn(rc, rc);
2561	return rc;
2562	}
2563
2564
2565	/**
2566	* Loads the guest's RIP into the guest-state area in the VMCS.
2567	*
2568	* @returns VBox status code.
2569	* @param pVM Pointer to the VM.
2570	* @param pVCpu Pointer to the VMCPU.
2571	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2572	* out-of-sync. Make sure to update the required fields
2573	* before using them.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	*/
2577	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2578	{
2579	int rc = VINF_SUCCESS;
2580	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2581	{
2582	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2583	AssertRCReturn(rc, rc);
2584	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2585	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2586	}
2587	return rc;
2588	}
2589
2590
2591	/**
2592	* Loads the guest's RSP into the guest-state area in the VMCS.
2593	*
2594	* @returns VBox status code.
2595	* @param pVM Pointer to the VM.
2596	* @param pVCpu Pointer to the VMCPU.
2597	* @param pCtx Pointer to the guest-CPU context.
2598	*
2599	* @remarks No-long-jump zone!!!
2600	*/
2601	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2602	{
2603	int rc = VINF_SUCCESS;
2604	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2605	{
2606	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2607	AssertRCReturn(rc, rc);
2608	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2609	}
2610	return rc;
2611	}
2612
2613
2614	/**
2615	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2616	*
2617	* @returns VBox status code.
2618	* @param pVM Pointer to the VM.
2619	* @param pVCpu Pointer to the VMCPU.
2620	* @param pCtx Pointer to the guest-CPU context.
2621	*
2622	* @remarks No-long-jump zone!!!
2623	*/
2624	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2625	{
2626	int rc = VINF_SUCCESS;
2627	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2628	{
2629	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2630	Let us assert it as such and use native-width VMWRITE. */
2631	X86RFLAGS uRFlags = pCtx->rflags;
2632	Assert(uRFlags.u64 >> 32 == 0);
2633	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2634	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2635
2636	/*
2637	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2638	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2639	*/
2640	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2641	{
2642	Assert(pVM->hm.s.vmx.pRealModeTSS);
2643	Assert(PDMVmmDevHeapIsEnabled(pVM));
2644	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2645	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2646	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2647	}
2648
2649	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2650	AssertRCReturn(rc, rc);
2651
2652	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2653	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2654	}
2655	return rc;
2656	}
2657
2658
2659	/**
2660	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2661	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2662	* assembly code.
2663	*
2664	* @returns VBox status code.
2665	* @param pVM Pointer to the VM.
2666	* @param pVCpu Pointer to the VMCPU.
2667	* @param pCtx Pointer to the guest-CPU context.
2668	*
2669	* @remarks No-long-jump zone!!!
2670	*/
2671	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2672	{
2673	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2674	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2675	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2676	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2677	return rc;
2678	}
2679
2680
2681	/**
2682	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2683	* in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVM Pointer to the VM.
2687	* @param pVCpu Pointer to the VMCPU.
2688	* @param pCtx Pointer to the guest-CPU context.
2689	*
2690	* @remarks No-long-jump zone!!!
2691	*/
2692	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2693	{
2694	int rc = VINF_SUCCESS;
2695
2696	/*
2697	* Guest CR0.
2698	* Guest FPU.
2699	*/
2700	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2701	{
2702	uint64_t u64GuestCR0 = pCtx->cr0;
2703
2704	/* The guest's view (read access) of its CR0 is unblemished. */
2705	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2706	AssertRCReturn(rc, rc);
2707	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2708
2709	/* Setup VT-x's view of the guest CR0. */
2710	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2711	if (pVM->hm.s.fNestedPaging)
2712	{
2713	if (CPUMIsGuestPagingEnabledEx(pCtx))
2714	{
2715	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2716	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2717	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2718	}
2719	else
2720	{
2721	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2722	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2723	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2724	}
2725
2726	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2727	AssertRCReturn(rc, rc);
2728	}
2729	else
2730	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2731
2732	/*
2733	* Guest FPU bits.
2734	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2735	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2736	*/
2737	u64GuestCR0 \|= X86_CR0_NE;
2738	bool fInterceptNM = false;
2739	if (CPUMIsGuestFPUStateActive(pVCpu))
2740	{
2741	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2742	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2743	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2744	}
2745	else
2746	{
2747	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2748	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2749	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2750	}
2751
2752	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2753	bool fInterceptMF = false;
2754	if (!(pCtx->cr0 & X86_CR0_NE))
2755	fInterceptMF = true;
2756
2757	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2758	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2759	{
2760	Assert(PDMVmmDevHeapIsEnabled(pVM));
2761	Assert(pVM->hm.s.vmx.pRealModeTSS);
2762	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2763	fInterceptNM = true;
2764	fInterceptMF = true;
2765	}
2766	else
2767	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2768
2769	if (fInterceptNM)
2770	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2771	else
2772	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2773
2774	if (fInterceptMF)
2775	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2776	else
2777	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2778
2779	/* Additional intercepts for debugging, define these yourself explicitly. */
2780	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2781	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2782	\| RT_BIT(X86_XCPT_DB)
2783	\| RT_BIT(X86_XCPT_DE)
2784	\| RT_BIT(X86_XCPT_NM)
2785	\| RT_BIT(X86_XCPT_UD)
2786	\| RT_BIT(X86_XCPT_NP)
2787	\| RT_BIT(X86_XCPT_SS)
2788	\| RT_BIT(X86_XCPT_GP)
2789	\| RT_BIT(X86_XCPT_PF)
2790	\| RT_BIT(X86_XCPT_MF);
2791	#elif defined(VBOX_ALWAYS_TRAP_PF)
2792	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2793	#endif
2794
2795	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2796
2797	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2798	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2799	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2800	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2801	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2802	else
2803	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2804
2805	u64GuestCR0 \|= uSetCR0;
2806	u64GuestCR0 &= uZapCR0;
2807	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2808
2809	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2810	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2811	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2812	Log(("Load: VMX_VMCS_GUEST_CR0=%RGv (uSetCR0=%#RGv uZapCR0=%#RGv)\n", u64GuestCR0, uSetCR0, uZapCR0));
2813
2814	/*
2815	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2816	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2817	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2818	*/
2819	uint64_t u64CR0Mask = 0;
2820	u64CR0Mask = X86_CR0_PE
2821	\| X86_CR0_NE
2822	\| X86_CR0_WP
2823	\| X86_CR0_PG
2824	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2825	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2826	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2827	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2828	u64CR0Mask &= ~X86_CR0_PE;
2829	/* Enable this later. */
2830	/* if (pVM->hm.s.fNestedPaging)
2831	u64CR0Mask &= ~X86_CR0_WP; */
2832
2833	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2834	if (fInterceptNM)
2835	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2836	else
2837	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2838
2839	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2840	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2841	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2842	AssertRCReturn(rc, rc);
2843
2844	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2845	}
2846
2847	/*
2848	* Guest CR2.
2849	* It's always loaded in the assembler code. Nothing to do here.
2850	*/
2851
2852	/*
2853	* Guest CR3.
2854	*/
2855	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2856	{
2857	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2858	if (pVM->hm.s.fNestedPaging)
2859	{
2860	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2861
2862	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2863	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2864	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2865	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2866
2867	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2868	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2869	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2870
2871	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2872	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2873	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2874	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2875
2876	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2877	AssertRCReturn(rc, rc);
2878	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2879
2880	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2881	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2882	{
2883	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2884	if (CPUMIsGuestInPAEModeEx(pCtx))
2885	{
2886	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2887	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2888	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2889	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2890	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2891	AssertRCReturn(rc, rc);
2892	}
2893
2894	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2895	have Unrestricted Execution to handle the guest when it's not using paging. */
2896	GCPhysGuestCR3 = pCtx->cr3;
2897	}
2898	else
2899	{
2900	/*
2901	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2902	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2903	* EPT takes care of translating it to host-physical addresses.
2904	*/
2905	RTGCPHYS GCPhys;
2906	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2907	Assert(PDMVmmDevHeapIsEnabled(pVM));
2908
2909	/* We obtain it here every time as the guest could have relocated this PCI region. */
2910	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2911	AssertRCReturn(rc, rc);
2912
2913	GCPhysGuestCR3 = GCPhys;
2914	}
2915	}
2916	else
2917	{
2918	/* Non-nested paging case, just use the hypervisor's CR3. */
2919	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2920	}
2921
2922	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2923	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2924	AssertRCReturn(rc, rc);
2925
2926	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2927	}
2928
2929	/*
2930	* Guest CR4.
2931	*/
2932	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2933	{
2934	uint32_t u64GuestCR4 = pCtx->cr4;
2935	Assert(!(pCtx->cr4 >> 32));
2936
2937	/* The guest's view of its CR4 is unblemished. */
2938	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2939	AssertRCReturn(rc, rc);
2940	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2941
2942	/* Setup VT-x's view of the guest CR4. */
2943	/*
2944	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2945	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2946	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2947	*/
2948	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2949	{
2950	Assert(pVM->hm.s.vmx.pRealModeTSS);
2951	Assert(PDMVmmDevHeapIsEnabled(pVM));
2952	u64GuestCR4 &= ~X86_CR4_VME;
2953	}
2954
2955	if (pVM->hm.s.fNestedPaging)
2956	{
2957	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2958	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2959	{
2960	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2961	u64GuestCR4 \|= X86_CR4_PSE;
2962	/* Our identity mapping is a 32 bits page directory. */
2963	u64GuestCR4 &= ~X86_CR4_PAE;
2964	}
2965	/* else use guest CR4.*/
2966	}
2967	else
2968	{
2969	/*
2970	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2971	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2972	*/
2973	switch (pVCpu->hm.s.enmShadowMode)
2974	{
2975	case PGMMODE_REAL: /* Real-mode. */
2976	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2977	case PGMMODE_32_BIT: /* 32-bit paging. */
2978	{
2979	u64GuestCR4 &= ~X86_CR4_PAE;
2980	break;
2981	}
2982
2983	case PGMMODE_PAE: /* PAE paging. */
2984	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2985	{
2986	u64GuestCR4 \|= X86_CR4_PAE;
2987	break;
2988	}
2989
2990	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2991	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2992	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2993	break;
2994	#endif
2995	default:
2996	AssertFailed();
2997	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2998	}
2999	}
3000
3001	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3002	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3003	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3004	u64GuestCR4 \|= uSetCR4;
3005	u64GuestCR4 &= uZapCR4;
3006
3007	/* Write VT-x's view of the guest CR4 into the VMCS. */
3008	Log(("Load: VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
3009	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
3010
3011	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3012	uint64_t u64CR4Mask = 0;
3013	u64CR4Mask = X86_CR4_VME
3014	\| X86_CR4_PAE
3015	\| X86_CR4_PGE
3016	\| X86_CR4_PSE
3017	\| X86_CR4_VMXE;
3018	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3019	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3020	AssertRCReturn(rc, rc);
3021
3022	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3023	}
3024	return rc;
3025	}
3026
3027
3028	/**
3029	* Loads the guest debug registers into the guest-state area in the VMCS.
3030	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3031	*
3032	* @returns VBox status code.
3033	* @param pVM Pointer to the VM.
3034	* @param pVCpu Pointer to the VMCPU.
3035	* @param pCtx Pointer to the guest-CPU context.
3036	*
3037	* @remarks No-long-jump zone!!!
3038	*/
3039	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3040	{
3041	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3042	return VINF_SUCCESS;
3043
3044	#ifdef VBOX_STRICT
3045	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3046	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3047	{
3048	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3049	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3050	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3051	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3052	}
3053	#endif
3054
3055	int rc = VERR_INTERNAL_ERROR_5;
3056	bool fInterceptDB = false;
3057	bool fInterceptMovDRx = false;
3058	if (DBGFIsStepping(pVCpu))
3059	{
3060	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3061	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3062	{
3063	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3064	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3065	AssertRCReturn(rc, rc);
3066	Assert(fInterceptDB == false);
3067	}
3068	else
3069	fInterceptDB = true;
3070	}
3071
3072	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3073	{
3074	if (!CPUMIsHyperDebugStateActive(pVCpu))
3075	{
3076	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3077	AssertRC(rc);
3078	}
3079	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3080	fInterceptMovDRx = true;
3081	}
3082	else if (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3083	{
3084	if (!CPUMIsGuestDebugStateActive(pVCpu))
3085	{
3086	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3087	AssertRC(rc);
3088	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3089	}
3090	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3091	Assert(fInterceptMovDRx == false);
3092	}
3093	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3094	{
3095	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3096	fInterceptMovDRx = true;
3097	}
3098
3099	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3100	if (fInterceptDB)
3101	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3102	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3103	{
3104	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3105	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3106	#endif
3107	}
3108
3109	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3110	if (fInterceptMovDRx)
3111	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3112	else
3113	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3114
3115	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3116	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3117
3118	/* The guest's view of its DR7 is unblemished. */
3119	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3120
3121	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3122	return rc;
3123	}
3124
3125
3126	#ifdef VBOX_STRICT
3127	/**
3128	* Strict function to validate segment registers.
3129	*/
3130	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3131	{
3132	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3133	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3134	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3135	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3136	{
3137	/* Protected mode checks */
3138	/* CS */
3139	Assert(pCtx->cs.Attr.n.u1Present);
3140	Assert(!(pCtx->cs.Attr.u & 0xf00));
3141	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3142	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3143	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3144	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3145	\|\| (pCtx->cs.Attr.n.u1Granularity));
3146	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3147	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3148	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3149	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3150	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3151	else
3152	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3153	/* SS */
3154	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3155	{
3156	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3157	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3158	Assert(pCtx->ss.Attr.n.u1Present);
3159	Assert(!(pCtx->ss.Attr.u & 0xf00));
3160	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3161	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3162	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3163	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3164	\|\| (pCtx->ss.Attr.n.u1Granularity));
3165	}
3166	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3167	/* CR0 might not be up-to-date here always, hence disabled. */
3168	#if 0
3169	if (!pCtx->cr0 & X86_CR0_PE)
3170	Assert(!pCtx->ss.Attr.n.u2Dpl);
3171	#endif
3172	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3173	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3174	{
3175	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3176	Assert(pCtx->ds.Attr.n.u1Present);
3177	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3178	Assert(!(pCtx->ds.Attr.u & 0xf00));
3179	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3180	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3181	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3182	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3183	\|\| (pCtx->ds.Attr.n.u1Granularity));
3184	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3185	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3186	}
3187	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3188	{
3189	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3190	Assert(pCtx->es.Attr.n.u1Present);
3191	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3192	Assert(!(pCtx->es.Attr.u & 0xf00));
3193	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3194	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3195	\|\| !(pCtx->es.Attr.n.u1Granularity));
3196	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3197	\|\| (pCtx->es.Attr.n.u1Granularity));
3198	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3199	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3200	}
3201	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3202	{
3203	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3204	Assert(pCtx->fs.Attr.n.u1Present);
3205	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3206	Assert(!(pCtx->fs.Attr.u & 0xf00));
3207	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3208	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3209	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3210	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3211	\|\| (pCtx->fs.Attr.n.u1Granularity));
3212	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3213	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3214	}
3215	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3216	{
3217	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3218	Assert(pCtx->gs.Attr.n.u1Present);
3219	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3220	Assert(!(pCtx->gs.Attr.u & 0xf00));
3221	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3222	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3223	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3224	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3225	\|\| (pCtx->gs.Attr.n.u1Granularity));
3226	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3227	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3228	}
3229	/* 64-bit capable CPUs. */
3230	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3231	Assert(!(pCtx->cs.u64Base >> 32));
3232	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3233	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3234	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3235	# endif
3236	}
3237	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3238	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3239	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3240	{
3241	/* Real and v86 mode checks. */
3242	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3243	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3244	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3245	{
3246	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3247	}
3248	else
3249	{
3250	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3251	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3252	}
3253
3254	/* CS */
3255	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3256	Assert(pCtx->cs.u32Limit == 0xffff);
3257	Assert(u32CSAttr == 0xf3);
3258	/* SS */
3259	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3260	Assert(pCtx->ss.u32Limit == 0xffff);
3261	Assert(u32SSAttr == 0xf3);
3262	/* DS */
3263	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3264	Assert(pCtx->ds.u32Limit == 0xffff);
3265	Assert(u32DSAttr == 0xf3);
3266	/* ES */
3267	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3268	Assert(pCtx->es.u32Limit == 0xffff);
3269	Assert(u32ESAttr == 0xf3);
3270	/* FS */
3271	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3272	Assert(pCtx->fs.u32Limit == 0xffff);
3273	Assert(u32FSAttr == 0xf3);
3274	/* GS */
3275	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3276	Assert(pCtx->gs.u32Limit == 0xffff);
3277	Assert(u32GSAttr == 0xf3);
3278	/* 64-bit capable CPUs. */
3279	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3280	Assert(!(pCtx->cs.u64Base >> 32));
3281	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3282	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3283	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3284	# endif
3285	}
3286	}
3287	#endif /* VBOX_STRICT */
3288
3289
3290	/**
3291	* Writes a guest segment register into the guest-state area in the VMCS.
3292	*
3293	* @returns VBox status code.
3294	* @param pVM Pointer to the VM.
3295	* @param pVCpu Pointer to the VMCPU.
3296	* @param idxSel Index of the selector in the VMCS.
3297	* @param idxLimit Index of the segment limit in the VMCS.
3298	* @param idxBase Index of the segment base in the VMCS.
3299	* @param idxAccess Index of the access rights of the segment in the VMCS.
3300	* @param pSelReg Pointer to the segment selector.
3301	* @param pCtx Pointer to the guest-CPU context.
3302	*
3303	* @remarks No-long-jump zone!!!
3304	*/
3305	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3306	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3307	{
3308	int rc;
3309	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3310	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3311	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3312	AssertRCReturn(rc, rc);
3313
3314	uint32_t u32Access = pSelReg->Attr.u;
3315	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3316	{
3317	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3318	u32Access = 0xf3;
3319	Assert(pVM->hm.s.vmx.pRealModeTSS);
3320	Assert(PDMVmmDevHeapIsEnabled(pVM));
3321	}
3322	else
3323	{
3324	/*
3325	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3326	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3327	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3328	* loaded in protected-mode have their attribute as 0.
3329	*/
3330	if (!u32Access)
3331	u32Access = HMVMX_SEL_UNUSABLE;
3332	}
3333
3334	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3335	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3336	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3337
3338	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3339	AssertRCReturn(rc, rc);
3340	return rc;
3341	}
3342
3343
3344	/**
3345	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3346	* into the guest-state area in the VMCS.
3347	*
3348	* @returns VBox status code.
3349	* @param pVM Pointer to the VM.
3350	* @param pVCPU Pointer to the VMCPU.
3351	* @param pCtx Pointer to the guest-CPU context.
3352	*
3353	* @remarks No-long-jump zone!!!
3354	*/
3355	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3356	{
3357	int rc = VERR_INTERNAL_ERROR_5;
3358
3359	/*
3360	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3361	*/
3362	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3363	{
3364	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3365	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3366	{
3367	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3368	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3369	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3370	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3371	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3372	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3373	}
3374
3375	#ifdef VBOX_WITH_REM
3376	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3377	{
3378	Assert(pVM->hm.s.vmx.pRealModeTSS);
3379	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3380	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3381	{
3382	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3383	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3384	&& enmGuestMode >= PGMMODE_PROTECTED)
3385	{
3386	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3387	in real-mode (e.g. OpenBSD 4.0) */
3388	REMFlushTBs(pVM);
3389	Log(("Load: Switch to protected mode detected!\n"));
3390	}
3391	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3392	}
3393	}
3394	#endif
3395	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3396	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3397	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3398	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3399	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3400	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3401	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3402	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3403	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3404	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3405	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3406	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3407	AssertRCReturn(rc, rc);
3408
3409	#ifdef VBOX_STRICT
3410	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3411	#endif
3412	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3413	}
3414
3415	/*
3416	* Guest TR.
3417	*/
3418	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3419	{
3420	/*
3421	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3422	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3423	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3424	*/
3425	uint16_t u16Sel = 0;
3426	uint32_t u32Limit = 0;
3427	uint64_t u64Base = 0;
3428	uint32_t u32AccessRights = 0;
3429
3430	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3431	{
3432	u16Sel = pCtx->tr.Sel;
3433	u32Limit = pCtx->tr.u32Limit;
3434	u64Base = pCtx->tr.u64Base;
3435	u32AccessRights = pCtx->tr.Attr.u;
3436	}
3437	else
3438	{
3439	Assert(pVM->hm.s.vmx.pRealModeTSS);
3440	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3441
3442	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3443	RTGCPHYS GCPhys;
3444	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3445	AssertRCReturn(rc, rc);
3446
3447	X86DESCATTR DescAttr;
3448	DescAttr.u = 0;
3449	DescAttr.n.u1Present = 1;
3450	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3451
3452	u16Sel = 0;
3453	u32Limit = HM_VTX_TSS_SIZE;
3454	u64Base = GCPhys; /* in real-mode phys = virt. */
3455	u32AccessRights = DescAttr.u;
3456	}
3457
3458	/* Validate. */
3459	Assert(!(u16Sel & RT_BIT(2)));
3460	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3461	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3462	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3463	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3464	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3465	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3466	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3467	Assert( (u32Limit & 0xfff) == 0xfff
3468	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3469	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3470	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3471
3472	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3473	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3474	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3475	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3476	AssertRCReturn(rc, rc);
3477
3478	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3479	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3480	}
3481
3482	/*
3483	* Guest GDTR.
3484	*/
3485	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3486	{
3487	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3488	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3489	AssertRCReturn(rc, rc);
3490
3491	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3492	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3493	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3494	}
3495
3496	/*
3497	* Guest LDTR.
3498	*/
3499	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3500	{
3501	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3502	uint32_t u32Access = 0;
3503	if (!pCtx->ldtr.Attr.u)
3504	u32Access = HMVMX_SEL_UNUSABLE;
3505	else
3506	u32Access = pCtx->ldtr.Attr.u;
3507
3508	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3509	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3510	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3511	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3512	AssertRCReturn(rc, rc);
3513
3514	/* Validate. */
3515	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3516	{
3517	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3518	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3519	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3520	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3521	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3522	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3523	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3524	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3525	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3526	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3527	}
3528
3529	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3530	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3531	}
3532
3533	/*
3534	* Guest IDTR.
3535	*/
3536	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3537	{
3538	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3539	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3540	AssertRCReturn(rc, rc);
3541
3542	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3543	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3544	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3545	}
3546
3547	return VINF_SUCCESS;
3548	}
3549
3550
3551	/**
3552	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3553	* areas. These MSRs will automatically be loaded to the host CPU on every
3554	* successful VM entry and stored from the host CPU on every successful VM exit.
3555	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3556	*
3557	* @returns VBox status code.
3558	* @param pVM Pointer to the VM.
3559	* @param pVCpu Pointer to the VMCPU.
3560	* @param pCtx Pointer to the guest-CPU context.
3561	*
3562	* @remarks No-long-jump zone!!!
3563	*/
3564	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3565	{
3566	AssertPtr(pVCpu);
3567	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3568
3569	/*
3570	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3571	*/
3572	int rc = VINF_SUCCESS;
3573	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3574	{
3575	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3576	uint32_t cGuestMsrs = 0;
3577
3578	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3579	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3580	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3581	if (fSupportsNX \|\| fSupportsLongMode)
3582	{
3583	/** @todo support save IA32_EFER, i.e.
3584	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3585	* guest EFER need not be part of the VM-entry MSR-load area. */
3586	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3587	pGuestMsr->u32Reserved = 0;
3588	pGuestMsr->u64Value = pCtx->msrEFER;
3589	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3590	if (!CPUMIsGuestInLongModeEx(pCtx))
3591	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3592	pGuestMsr++; cGuestMsrs++;
3593	if (fSupportsLongMode)
3594	{
3595	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3596	pGuestMsr->u32Reserved = 0;
3597	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3598	pGuestMsr++; cGuestMsrs++;
3599	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3600	pGuestMsr->u32Reserved = 0;
3601	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3602	pGuestMsr++; cGuestMsrs++;
3603	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3604	pGuestMsr->u32Reserved = 0;
3605	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3606	pGuestMsr++; cGuestMsrs++;
3607	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3608	#if 0
3609	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3610	pGuestMsr->u32Reserved = 0;
3611	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3612	pGuestMsr++; cGuestMsrs++;
3613	#endif
3614	}
3615	}
3616
3617	/*
3618	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3619	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3620	*/
3621	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3622	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3623	{
3624	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3625	pGuestMsr->u32Reserved = 0;
3626	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3627	AssertRCReturn(rc, rc);
3628	pGuestMsr++; cGuestMsrs++;
3629	}
3630
3631	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3632	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3633	{
3634	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3635	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3636	}
3637
3638	/* Update the VCPU's copy of the guest MSR count. */
3639	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3640	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3641	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3642	AssertRCReturn(rc, rc);
3643
3644	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3645	}
3646
3647	/*
3648	* Guest Sysenter MSRs.
3649	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3650	* VM exits on WRMSRs for these MSRs.
3651	*/
3652	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3653	{
3654	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3656	}
3657	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3658	{
3659	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3661	}
3662	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3663	{
3664	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3665	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3666	}
3667	AssertRCReturn(rc, rc);
3668
3669	return rc;
3670	}
3671
3672
3673	/**
3674	* Loads the guest activity state into the guest-state area in the VMCS.
3675	*
3676	* @returns VBox status code.
3677	* @param pVM Pointer to the VM.
3678	* @param pVCpu Pointer to the VMCPU.
3679	* @param pCtx Pointer to the guest-CPU context.
3680	*
3681	* @remarks No-long-jump zone!!!
3682	*/
3683	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3684	{
3685	/** @todo See if we can make use of other states, e.g.
3686	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3687	int rc = VINF_SUCCESS;
3688	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3689	{
3690	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3691	AssertRCReturn(rc, rc);
3692	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3693	}
3694	return rc;
3695	}
3696
3697
3698	/**
3699	* Sets up the appropriate function to run guest code.
3700	*
3701	* @returns VBox status code.
3702	* @param pVM Pointer to the VM.
3703	* @param pVCpu Pointer to the VMCPU.
3704	* @param pCtx Pointer to the guest-CPU context.
3705	*
3706	* @remarks No-long-jump zone!!!
3707	*/
3708	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3709	{
3710	if (CPUMIsGuestInLongModeEx(pCtx))
3711	{
3712	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3713	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3714	#endif
3715	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3716	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3717	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3718	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3719	#else
3720	/* 64-bit host or hybrid host. */
3721	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3722	#endif
3723	}
3724	else
3725	{
3726	/* Guest is not in long mode, use the 32-bit handler. */
3727	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3728	}
3729	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3730	return VINF_SUCCESS;
3731	}
3732
3733
3734	/**
3735	* Wrapper for running the guest code in VT-x.
3736	*
3737	* @returns VBox strict status code.
3738	* @param pVM Pointer to the VM.
3739	* @param pVCpu Pointer to the VMCPU.
3740	* @param pCtx Pointer to the guest-CPU context.
3741	*
3742	* @remarks No-long-jump zone!!!
3743	*/
3744	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3745	{
3746	/*
3747	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3748	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3749	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3750	*/
3751	#ifdef VBOX_WITH_KERNEL_USING_XMM
3752	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3753	#else
3754	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3755	#endif
3756	}
3757
3758
3759	/**
3760	* Report world-switch error and dump some useful debug info.
3761	*
3762	* @param pVM Pointer to the VM.
3763	* @param pVCpu Pointer to the VMCPU.
3764	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3765	* @param pCtx Pointer to the guest-CPU context.
3766	* @param pVmxTransient Pointer to the VMX transient structure (only
3767	* exitReason updated).
3768	*/
3769	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3770	{
3771	Assert(pVM);
3772	Assert(pVCpu);
3773	Assert(pCtx);
3774	Assert(pVmxTransient);
3775	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3776
3777	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3778	switch (rcVMRun)
3779	{
3780	case VERR_VMX_INVALID_VMXON_PTR:
3781	AssertFailed();
3782	break;
3783	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3784	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3785	{
3786	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3787	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3788	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3789	AssertRC(rc);
3790
3791	#ifdef VBOX_STRICT
3792	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3793	pVmxTransient->uExitReason));
3794	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3795	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3796	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3797	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3798	else
3799	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3800
3801	/* VMX control bits. */
3802	uint32_t u32Val;
3803	uint64_t u64Val;
3804	RTHCUINTREG uHCReg;
3805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3806	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3808	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3810	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3837	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3838	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3839	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3840	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3841	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3842	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3843	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3844	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3845	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3846	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3847
3848	/* Guest bits. */
3849	RTGCUINTREG uGCReg;
3850	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3851	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3852	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3853	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3854	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3855	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3856	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3857	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3858
3859	/* Host bits. */
3860	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3861	Log(("Host CR0 %#RHr\n", uHCReg));
3862	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3863	Log(("Host CR3 %#RHr\n", uHCReg));
3864	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3865	Log(("Host CR4 %#RHr\n", uHCReg));
3866
3867	RTGDTR HostGdtr;
3868	PCX86DESCHC pDesc;
3869	ASMGetGDTR(&HostGdtr);
3870	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3871	Log(("Host CS %#08x\n", u32Val));
3872	if (u32Val < HostGdtr.cbGdt)
3873	{
3874	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3875	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3876	}
3877
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3879	Log(("Host DS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3887	Log(("Host ES %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3892	}
3893
3894	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3895	Log(("Host FS %#08x\n", u32Val));
3896	if (u32Val < HostGdtr.cbGdt)
3897	{
3898	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3899	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3900	}
3901
3902	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3903	Log(("Host GS %#08x\n", u32Val));
3904	if (u32Val < HostGdtr.cbGdt)
3905	{
3906	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3907	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3908	}
3909
3910	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3911	Log(("Host SS %#08x\n", u32Val));
3912	if (u32Val < HostGdtr.cbGdt)
3913	{
3914	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3915	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3916	}
3917
3918	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3919	Log(("Host TR %#08x\n", u32Val));
3920	if (u32Val < HostGdtr.cbGdt)
3921	{
3922	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3923	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3924	}
3925
3926	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3927	Log(("Host TR Base %#RHv\n", uHCReg));
3928	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3929	Log(("Host GDTR Base %#RHv\n", uHCReg));
3930	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3931	Log(("Host IDTR Base %#RHv\n", uHCReg));
3932	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3933	Log(("Host SYSENTER CS %#08x\n", u32Val));
3934	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3935	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3936	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3937	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3939	Log(("Host RSP %#RHv\n", uHCReg));
3940	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3941	Log(("Host RIP %#RHv\n", uHCReg));
3942	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3943	if (HMVMX_IS_64BIT_HOST_MODE())
3944	{
3945	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3946	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3947	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3948	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3949	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3950	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3951	}
3952	# endif
3953	#endif /* VBOX_STRICT */
3954	break;
3955	}
3956
3957	default:
3958	/* Impossible */
3959	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3960	break;
3961	}
3962	NOREF(pVM);
3963	}
3964
3965
3966	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3967	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3968	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3969	#endif
3970
3971	#ifdef VBOX_STRICT
3972	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3973	{
3974	switch (idxField)
3975	{
3976	case VMX_VMCS_GUEST_RIP:
3977	case VMX_VMCS_GUEST_RSP:
3978	case VMX_VMCS_GUEST_RFLAGS:
3979	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3980	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3981	case VMX_VMCS_GUEST_CR0:
3982	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3983	case VMX_VMCS_GUEST_CR4:
3984	case VMX_VMCS_GUEST_DR7:
3985	case VMX_VMCS32_GUEST_SYSENTER_CS:
3986	case VMX_VMCS_GUEST_SYSENTER_EIP:
3987	case VMX_VMCS_GUEST_SYSENTER_ESP:
3988	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3989	case VMX_VMCS_GUEST_GDTR_BASE:
3990	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3991	case VMX_VMCS_GUEST_IDTR_BASE:
3992	case VMX_VMCS16_GUEST_FIELD_CS:
3993	case VMX_VMCS32_GUEST_CS_LIMIT:
3994	case VMX_VMCS_GUEST_CS_BASE:
3995	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3996	case VMX_VMCS16_GUEST_FIELD_DS:
3997	case VMX_VMCS32_GUEST_DS_LIMIT:
3998	case VMX_VMCS_GUEST_DS_BASE:
3999	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4000	case VMX_VMCS16_GUEST_FIELD_ES:
4001	case VMX_VMCS32_GUEST_ES_LIMIT:
4002	case VMX_VMCS_GUEST_ES_BASE:
4003	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4004	case VMX_VMCS16_GUEST_FIELD_FS:
4005	case VMX_VMCS32_GUEST_FS_LIMIT:
4006	case VMX_VMCS_GUEST_FS_BASE:
4007	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4008	case VMX_VMCS16_GUEST_FIELD_GS:
4009	case VMX_VMCS32_GUEST_GS_LIMIT:
4010	case VMX_VMCS_GUEST_GS_BASE:
4011	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4012	case VMX_VMCS16_GUEST_FIELD_SS:
4013	case VMX_VMCS32_GUEST_SS_LIMIT:
4014	case VMX_VMCS_GUEST_SS_BASE:
4015	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4016	case VMX_VMCS16_GUEST_FIELD_LDTR:
4017	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4018	case VMX_VMCS_GUEST_LDTR_BASE:
4019	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4020	case VMX_VMCS16_GUEST_FIELD_TR:
4021	case VMX_VMCS32_GUEST_TR_LIMIT:
4022	case VMX_VMCS_GUEST_TR_BASE:
4023	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4024	case VMX_VMCS32_RO_EXIT_REASON:
4025	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4026	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4027	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4028	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4029	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4030	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4031	case VMX_VMCS32_RO_IDT_INFO:
4032	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4033	case VMX_VMCS_GUEST_CR3:
4034	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4035	return true;
4036	}
4037	return false;
4038	}
4039
4040	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4041	{
4042	switch (idxField)
4043	{
4044	case VMX_VMCS_GUEST_LDTR_BASE:
4045	case VMX_VMCS_GUEST_TR_BASE:
4046	case VMX_VMCS_GUEST_GDTR_BASE:
4047	case VMX_VMCS_GUEST_IDTR_BASE:
4048	case VMX_VMCS_GUEST_SYSENTER_EIP:
4049	case VMX_VMCS_GUEST_SYSENTER_ESP:
4050	case VMX_VMCS_GUEST_CR0:
4051	case VMX_VMCS_GUEST_CR4:
4052	case VMX_VMCS_GUEST_CR3:
4053	case VMX_VMCS_GUEST_DR7:
4054	case VMX_VMCS_GUEST_RIP:
4055	case VMX_VMCS_GUEST_RSP:
4056	case VMX_VMCS_GUEST_CS_BASE:
4057	case VMX_VMCS_GUEST_DS_BASE:
4058	case VMX_VMCS_GUEST_ES_BASE:
4059	case VMX_VMCS_GUEST_FS_BASE:
4060	case VMX_VMCS_GUEST_GS_BASE:
4061	case VMX_VMCS_GUEST_SS_BASE:
4062	return true;
4063	}
4064	return false;
4065	}
4066	#endif /* VBOX_STRICT */
4067
4068	/**
4069	* Executes the specified handler in 64-bit mode.
4070	*
4071	* @returns VBox status code.
4072	* @param pVM Pointer to the VM.
4073	* @param pVCpu Pointer to the VMCPU.
4074	* @param pCtx Pointer to the guest CPU context.
4075	* @param pfnHandler Pointer to the RC handler function.
4076	* @param cbParam Number of parameters.
4077	* @param paParam Array of 32-bit parameters.
4078	*/
4079	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4080	uint32_t *paParam)
4081	{
4082	int rc, rc2;
4083	PHMGLOBLCPUINFO pCpu;
4084	RTHCPHYS HCPhysCpuPage;
4085	RTHCUINTREG uOldEFlags;
4086
4087	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4088	Assert(pfnHandler);
4089	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4090	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4091
4092	#ifdef VBOX_STRICT
4093	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4094	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4095
4096	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4097	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4098	#endif
4099
4100	/* Disable interrupts. */
4101	uOldEFlags = ASMIntDisableFlags();
4102
4103	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4104	RTCPUID idHostCpu = RTMpCpuId();
4105	CPUMR0SetLApic(pVM, idHostCpu);
4106	#endif
4107
4108	pCpu = HMR0GetCurrentCpu();
4109	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4110
4111	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4112	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4113
4114	/* Leave VMX Root Mode. */
4115	VMXDisable();
4116
4117	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4118
4119	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4120	CPUMSetHyperEIP(pVCpu, pfnHandler);
4121	for (int i = (int)cbParam - 1; i >= 0; i--)
4122	CPUMPushHyper(pVCpu, paParam[i]);
4123
4124	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4125
4126	/* Call the switcher. */
4127	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4128	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4129
4130	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4131	/* Make sure the VMX instructions don't cause #UD faults. */
4132	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4133
4134	/* Re-enter VMX Root Mode */
4135	rc2 = VMXEnable(HCPhysCpuPage);
4136	if (RT_FAILURE(rc2))
4137	{
4138	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4139	ASMSetFlags(uOldEFlags);
4140	return rc2;
4141	}
4142
4143	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4144	AssertRC(rc2);
4145	Assert(!(ASMGetFlags() & X86_EFL_IF));
4146	ASMSetFlags(uOldEFlags);
4147	return rc;
4148	}
4149
4150
4151	/**
4152	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4153	* supporting 64-bit guests.
4154	*
4155	* @returns VBox status code.
4156	* @param fResume Whether to VMLAUNCH or VMRESUME.
4157	* @param pCtx Pointer to the guest-CPU context.
4158	* @param pCache Pointer to the VMCS cache.
4159	* @param pVM Pointer to the VM.
4160	* @param pVCpu Pointer to the VMCPU.
4161	*/
4162	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4163	{
4164	uint32_t aParam[6];
4165	PHMGLOBLCPUINFO pCpu = NULL;
4166	RTHCPHYS HCPhysCpuPage = 0;
4167	int rc = VERR_INTERNAL_ERROR_5;
4168
4169	pCpu = HMR0GetCurrentCpu();
4170	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4171
4172	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4173	pCache->uPos = 1;
4174	pCache->interPD = PGMGetInterPaeCR3(pVM);
4175	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4176	#endif
4177
4178	#ifdef VBOX_STRICT
4179	pCache->TestIn.HCPhysCpuPage = 0;
4180	pCache->TestIn.HCPhysVmcs = 0;
4181	pCache->TestIn.pCache = 0;
4182	pCache->TestOut.HCPhysVmcs = 0;
4183	pCache->TestOut.pCache = 0;
4184	pCache->TestOut.pCtx = 0;
4185	pCache->TestOut.eflags = 0;
4186	#endif
4187
4188	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4189	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4190	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4191	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4192	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4193	aParam[5] = 0;
4194
4195	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4196	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4197	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4198	#endif
4199	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4200
4201	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4202	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4203	Assert(pCtx->dr[4] == 10);
4204	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4205	#endif
4206
4207	#ifdef VBOX_STRICT
4208	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4209	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4210	pVCpu->hm.s.vmx.HCPhysVmcs));
4211	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4212	pCache->TestOut.HCPhysVmcs));
4213	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4214	pCache->TestOut.pCache));
4215	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4216	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4217	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4218	pCache->TestOut.pCtx));
4219	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4220	#endif
4221	return rc;
4222	}
4223
4224
4225	/**
4226	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4227	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4228	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4229	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4230	*
4231	* @returns VBox status code.
4232	* @param pVM Pointer to the VM.
4233	* @param pVCpu Pointer to the VMCPU.
4234	*/
4235	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4236	{
4237	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4238	{ \
4239	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4240	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4241	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4242	}
4243
4244	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4245	{ \
4246	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4250	}
4251
4252	AssertPtr(pVM);
4253	AssertPtr(pVCpu);
4254	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4255
4256	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4257	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4258	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4259	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4260	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4261	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4262	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4263	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4264	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4265
4266	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4267	#if 0
4268	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4277	#endif
4278
4279	/* 32-bit guest-state fields. */
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4284	/* Unused 32-bit guest-state fields. */
4285	#if 0
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4289	#endif
4290
4291	/* Natural width guest-state fields. */
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4308	/* Unused natural width guest-state fields. */
4309	#if 0
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4312	#endif
4313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4315
4316	if (pVM->hm.s.fNestedPaging)
4317	{
4318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4320	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4321	}
4322	else
4323	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4324
4325	#undef VMXLOCAL_INIT_VMCS_SELREG
4326	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4327	return VINF_SUCCESS;
4328	}
4329
4330
4331	/**
4332	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4333	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4334	* darwin, running 64-bit guests).
4335	*
4336	* @returns VBox status code.
4337	* @param pVCpu Pointer to the VMCPU.
4338	* @param idxField The VMCS field encoding.
4339	* @param u64Val 16, 32 or 64 bits value.
4340	*/
4341	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4342	{
4343	int rc;
4344	switch (idxField)
4345	{
4346	/*
4347	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4348	*/
4349	/* 64-bit Control fields. */
4350	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4351	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4352	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4353	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4354	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4355	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4356	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4357	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4358	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4359	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4360	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4361	case VMX_VMCS64_CTRL_EPTP_FULL:
4362	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4363	/* 64-bit Read-only data fields. */
4364	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4365	/* 64-bit Guest-state fields. */
4366	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4367	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4368	case VMX_VMCS64_GUEST_PAT_FULL:
4369	case VMX_VMCS64_GUEST_EFER_FULL:
4370	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4371	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4372	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4373	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4374	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4375	/* 64-bit Host-state fields. */
4376	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4377	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4378	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4379	{
4380	rc = VMXWriteVmcs32(idxField, u64Val);
4381	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4382	break;
4383	}
4384
4385	/*
4386	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4387	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4388	*/
4389	/* Natural-width Guest-state fields. */
4390	case VMX_VMCS_GUEST_CR0:
4391	case VMX_VMCS_GUEST_CR3:
4392	case VMX_VMCS_GUEST_CR4:
4393	case VMX_VMCS_GUEST_ES_BASE:
4394	case VMX_VMCS_GUEST_CS_BASE:
4395	case VMX_VMCS_GUEST_SS_BASE:
4396	case VMX_VMCS_GUEST_DS_BASE:
4397	case VMX_VMCS_GUEST_FS_BASE:
4398	case VMX_VMCS_GUEST_GS_BASE:
4399	case VMX_VMCS_GUEST_LDTR_BASE:
4400	case VMX_VMCS_GUEST_TR_BASE:
4401	case VMX_VMCS_GUEST_GDTR_BASE:
4402	case VMX_VMCS_GUEST_IDTR_BASE:
4403	case VMX_VMCS_GUEST_DR7:
4404	case VMX_VMCS_GUEST_RSP:
4405	case VMX_VMCS_GUEST_RIP:
4406	case VMX_VMCS_GUEST_RFLAGS:
4407	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4408	case VMX_VMCS_GUEST_SYSENTER_ESP:
4409	case VMX_VMCS_GUEST_SYSENTER_EIP:
4410	{
4411	if ((u64Val >> 32ULL) == 0)
4412	{
4413	/* If this field is 64-bit, VT-x will zero out the top bits. */
4414	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4415	}
4416	else
4417	{
4418	/* Assert that only the 32->64 switcher case should ever come here. */
4419	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4420	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4421	}
4422	break;
4423	}
4424
4425	default:
4426	{
4427	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4428	rc = VERR_INVALID_PARAMETER;
4429	break;
4430	}
4431	}
4432	AssertRCReturn(rc, rc);
4433	return rc;
4434	}
4435
4436
4437	/**
4438	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4439	* hosts (except darwin) for 64-bit guests.
4440	*
4441	* @param pVCpu Pointer to the VMCPU.
4442	* @param idxField The VMCS field encoding.
4443	* @param u64Val 16, 32 or 64 bits value.
4444	*/
4445	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4446	{
4447	AssertPtr(pVCpu);
4448	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4449
4450	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4451	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4452
4453	/* Make sure there are no duplicates. */
4454	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4455	{
4456	if (pCache->Write.aField[i] == idxField)
4457	{
4458	pCache->Write.aFieldVal[i] = u64Val;
4459	return VINF_SUCCESS;
4460	}
4461	}
4462
4463	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4464	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4465	pCache->Write.cValidEntries++;
4466	return VINF_SUCCESS;
4467	}
4468
4469
4470	/**
4471	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4472	*
4473	* @param pVCpu Pointer to the VMCPU.
4474	* @param pCache Pointer to the VMCS cache.
4475	*
4476	* @remarks No-long-jump zone!!!
4477	*/
4478	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4479	{
4480	AssertPtr(pCache);
4481	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4482	{
4483	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4484	AssertRC(rc);
4485	}
4486	pCache->Write.cValidEntries = 0;
4487	}
4488
4489
4490	/**
4491	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4492	*
4493	* @param pVCpu Pointer to the VMCPU.
4494	* @param pCache Pointer to the VMCS cache.
4495	*
4496	* @remarks No-long-jump zone!!!
4497	*/
4498	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4499	{
4500	AssertPtr(pCache);
4501	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4502	{
4503	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4504	AssertRC(rc);
4505	}
4506	}
4507	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4508
4509
4510	/**
4511	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4512	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4513	* timer.
4514	*
4515	* @returns VBox status code.
4516	* @param pVM Pointer to the VM.
4517	* @param pVCpu Pointer to the VMCPU.
4518	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4519	* out-of-sync. Make sure to update the required fields
4520	* before using them.
4521	* @remarks No-long-jump zone!!!
4522	*/
4523	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4524	{
4525	int rc = VERR_INTERNAL_ERROR_5;
4526	bool fOffsettedTsc = false;
4527	if (pVM->hm.s.vmx.fUsePreemptTimer)
4528	{
4529	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4530
4531	/* Make sure the returned values have sane upper and lower boundaries. */
4532	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4533	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4534	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4535	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4536
4537	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4538	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4539	}
4540	else
4541	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4542
4543	if (fOffsettedTsc)
4544	{
4545	uint64_t u64CurTSC = ASMReadTSC();
4546	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4547	{
4548	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4549	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4550
4551	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4552	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4553	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4554	}
4555	else
4556	{
4557	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4558	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4559	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4560	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4561	}
4562	}
4563	else
4564	{
4565	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4566	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4567	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4568	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4569	}
4570	}
4571
4572
4573	/**
4574	* Determines if an exception is a contributory exception. Contributory
4575	* exceptions are ones which can cause double-faults.
4576	*
4577	* @returns true if the exception is contributory, false otherwise.
4578	* @param uVector The exception vector.
4579	*/
4580	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4581	{
4582	switch (uVector)
4583	{
4584	case X86_XCPT_GP:
4585	case X86_XCPT_SS:
4586	case X86_XCPT_NP:
4587	case X86_XCPT_TS:
4588	case X86_XCPT_DE:
4589	return true;
4590	default:
4591	break;
4592	}
4593	return false;
4594	}
4595
4596
4597	/**
4598	* Determines if an exception is a benign exception. Benign exceptions
4599	* are ones which cannot cause double-faults.
4600	*
4601	* @returns true if the exception is benign, false otherwise.
4602	* @param uVector The exception vector.
4603	*/
4604	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4605	{
4606	if ( uVector == X86_XCPT_PF
4607	\|\| uVector == X86_XCPT_DF)
4608	{
4609	return false;
4610	}
4611
4612	bool fIsBenignXcpt = !hmR0VmxIsContributoryXcpt(uVector);
4613	#ifdef VBOX_STRICT
4614	switch (uVector)
4615	{
4616	case X86_XCPT_DB:
4617	case X86_XCPT_NMI:
4618	case X86_XCPT_BP:
4619	case X86_XCPT_OF:
4620	case X86_XCPT_BR:
4621	case X86_XCPT_UD:
4622	case X86_XCPT_NM:
4623	case X86_XCPT_CO_SEG_OVERRUN:
4624	case X86_XCPT_MF:
4625	case X86_XCPT_AC:
4626	case X86_XCPT_MC:
4627	case X86_XCPT_XF:
4628	AssertMsg(fIsBenignXcpt, ("%#x\n", uVector));
4629	break;
4630	default:
4631	AssertMsg(!fIsBenignXcpt, ("%#x\n", uVector));
4632	break;
4633	}
4634	#endif
4635	return fIsBenignXcpt;
4636	}
4637
4638
4639	/**
4640	* Sets an event as a pending event to be injected into the guest.
4641	*
4642	* @param pVCpu Pointer to the VMCPU.
4643	* @param u32IntrInfo The VM-entry interruption-information field.
4644	* @param cbInstr The VM-entry instruction length in bytes (for software
4645	* interrupts, exceptions and privileged software
4646	* exceptions).
4647	* @param u32ErrCode The VM-entry exception error code.
4648	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4649	* page-fault.
4650	*/
4651	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4652	RTGCUINTPTR GCPtrFaultAddress)
4653	{
4654	Assert(!pVCpu->hm.s.Event.fPending);
4655	pVCpu->hm.s.Event.fPending = true;
4656	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4657	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4658	pVCpu->hm.s.Event.cbInstr = cbInstr;
4659	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4660	}
4661
4662
4663	/**
4664	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4665	*
4666	* @param pVCpu Pointer to the VMCPU.
4667	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4668	* out-of-sync. Make sure to update the required fields
4669	* before using them.
4670	*/
4671	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4672	{
4673	/* Inject the double-fault. */
4674	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4675	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4676	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4677	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4678	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4679	}
4680
4681
4682	/**
4683	* Handle a condition that occurred while delivering an event through the guest
4684	* IDT.
4685	*
4686	* @returns VBox status code (informational error codes included).
4687	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4688	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4689	* continue execution of the guest which will delivery the #DF.
4690	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4691	*
4692	* @param pVCpu Pointer to the VMCPU.
4693	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4694	* out-of-sync. Make sure to update the required fields
4695	* before using them.
4696	* @param pVmxTransient Pointer to the VMX transient structure.
4697	*
4698	* @remarks No-long-jump zone!!!
4699	*/
4700	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4701	{
4702	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4703	AssertRC(rc);
4704	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4705	{
4706	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4707	AssertRCReturn(rc, rc);
4708
4709	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4710	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4711	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4712
4713	typedef enum
4714	{
4715	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4716	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4717	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4718	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4719	} VMXREFLECTXCPT;
4720
4721	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4722	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4723	RTGCUINTPTR GCPtrFaultAddress = 0;
4724	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4725	{
4726	enmReflect = VMXREFLECTXCPT_XCPT;
4727	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4728	&& uExitVector == X86_XCPT_PF)
4729	{
4730	GCPtrFaultAddress = pMixedCtx->cr2;
4731	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4732	}
4733	else if ( uExitVector == X86_XCPT_PF
4734	&& uIdtVector == X86_XCPT_PF)
4735	{
4736	pVmxTransient->fVectoringPF = true;
4737	GCPtrFaultAddress = pMixedCtx->cr2;
4738	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4739	}
4740	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4741	&& hmR0VmxIsContributoryXcpt(uExitVector)
4742	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4743	\|\| uIdtVector == X86_XCPT_PF))
4744	{
4745	enmReflect = VMXREFLECTXCPT_DF;
4746	}
4747	else if (uIdtVector == X86_XCPT_DF)
4748	enmReflect = VMXREFLECTXCPT_TF;
4749	}
4750	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4751	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4752	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4753	{
4754	/*
4755	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4756	* (whatever they are) as they reoccur when restarting the instruction.
4757	*/
4758	enmReflect = VMXREFLECTXCPT_XCPT;
4759	}
4760
4761	switch (enmReflect)
4762	{
4763	case VMXREFLECTXCPT_XCPT:
4764	{
4765	uint32_t u32ErrCode = 0;
4766	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4767	{
4768	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4769	AssertRCReturn(rc, rc);
4770	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4771	}
4772	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4773	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4774	rc = VINF_SUCCESS;
4775	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4776	break;
4777	}
4778
4779	case VMXREFLECTXCPT_DF:
4780	{
4781	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4782	rc = VINF_VMX_DOUBLE_FAULT;
4783	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4784	uExitVector));
4785	break;
4786	}
4787
4788	case VMXREFLECTXCPT_TF:
4789	{
4790	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4791	rc = VINF_EM_RESET;
4792	break;
4793	}
4794
4795	default:
4796	Assert(rc == VINF_SUCCESS);
4797	break;
4798	}
4799	}
4800	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4801	return rc;
4802	}
4803
4804
4805	/**
4806	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4807	*
4808	* @returns VBox status code.
4809	* @param pVCpu Pointer to the VMCPU.
4810	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4811	* out-of-sync. Make sure to update the required fields
4812	* before using them.
4813	*
4814	* @remarks No-long-jump zone!!!
4815	*/
4816	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4817	{
4818	int rc = VINF_SUCCESS;
4819	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0)
4820	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FPU))
4821	{
4822	RTGCUINTREG uVal = 0;
4823	RTCCUINTREG uShadow = 0;
4824	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4825	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4826	AssertRCReturn(rc, rc);
4827	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4828	CPUMSetGuestCR0(pVCpu, uVal);
4829	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0 \| HMVMX_UPDATED_GUEST_FPU;
4830	}
4831	return rc;
4832	}
4833
4834
4835	/**
4836	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4837	*
4838	* @returns VBox status code.
4839	* @param pVCpu Pointer to the VMCPU.
4840	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4841	* out-of-sync. Make sure to update the required fields
4842	* before using them.
4843	*
4844	* @remarks No-long-jump zone!!!
4845	*/
4846	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4847	{
4848	int rc = VINF_SUCCESS;
4849	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4850	{
4851	RTGCUINTREG uVal = 0;
4852	RTCCUINTREG uShadow = 0;
4853	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4854	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4855	AssertRCReturn(rc, rc);
4856	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4857	CPUMSetGuestCR4(pVCpu, uVal);
4858	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4859	}
4860	return rc;
4861	}
4862
4863
4864	/**
4865	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4866	*
4867	* @returns VBox status code.
4868	* @param pVCpu Pointer to the VMCPU.
4869	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4870	* out-of-sync. Make sure to update the required fields
4871	* before using them.
4872	*
4873	* @remarks No-long-jump zone!!!
4874	*/
4875	DECLINLINE(int) hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4876	{
4877	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP)
4878	return VINF_SUCCESS;
4879
4880	RTGCUINTREG uVal = 0;
4881	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4882	AssertRCReturn(rc, rc);
4883	pMixedCtx->rip = uVal;
4884	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4885	return rc;
4886	}
4887
4888
4889	/**
4890	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4891	*
4892	* @returns VBox status code.
4893	* @param pVCpu Pointer to the VMCPU.
4894	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4895	* out-of-sync. Make sure to update the required fields
4896	* before using them.
4897	*
4898	* @remarks No-long-jump zone!!!
4899	*/
4900	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4901	{
4902	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP)
4903	return VINF_SUCCESS;
4904
4905	RTGCUINTREG uVal = 0;
4906	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4907	AssertRCReturn(rc, rc);
4908	pMixedCtx->rsp = uVal;
4909	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4910	return rc;
4911	}
4912
4913
4914	/**
4915	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4916	*
4917	* @returns VBox status code.
4918	* @param pVCpu Pointer to the VMCPU.
4919	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4920	* out-of-sync. Make sure to update the required fields
4921	* before using them.
4922	*
4923	* @remarks No-long-jump zone!!!
4924	*/
4925	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4926	{
4927	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS)
4928	return VINF_SUCCESS;
4929
4930	RTGCUINTREG uVal = 0;
4931	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4932	AssertRCReturn(rc, rc);
4933	pMixedCtx->rflags.u64 = uVal;
4934
4935	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4936	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4937	{
4938	PVM pVM = pVCpu->CTX_SUFF(pVM);
4939	Assert(pVM->hm.s.vmx.pRealModeTSS);
4940	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4941	pMixedCtx->eflags.Bits.u1VM = 0;
4942	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4943	}
4944
4945	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4946	return rc;
4947	}
4948
4949
4950	/**
4951	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4952	* guest-CPU context.
4953	*/
4954	static int hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4955	{
4956	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4957	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4958	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4959	return rc;
4960	}
4961
4962
4963	/**
4964	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4965	* from the guest-state area in the VMCS.
4966	*
4967	* @param pVCpu Pointer to the VMCPU.
4968	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4969	* out-of-sync. Make sure to update the required fields
4970	* before using them.
4971	*
4972	* @remarks No-long-jump zone!!!
4973	*/
4974	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4975	{
4976	uint32_t uIntrState = 0;
4977	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4978	AssertRC(rc);
4979
4980	if (!uIntrState)
4981	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4982	else
4983	{
4984	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4985	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4986	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4987	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4988	AssertRC(rc);
4989	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4990	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4991	}
4992	}
4993
4994
4995	/**
4996	* Saves the guest's activity state.
4997	*
4998	* @returns VBox status code.
4999	* @param pVCpu Pointer to the VMCPU.
5000	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5001	* out-of-sync. Make sure to update the required fields
5002	* before using them.
5003	*
5004	* @remarks No-long-jump zone!!!
5005	*/
5006	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5007	{
5008	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5009	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
5010	return VINF_SUCCESS;
5011	}
5012
5013
5014	/**
5015	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5016	* the current VMCS into the guest-CPU context.
5017	*
5018	* @returns VBox status code.
5019	* @param pVCpu Pointer to the VMCPU.
5020	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5021	* out-of-sync. Make sure to update the required fields
5022	* before using them.
5023	*
5024	* @remarks No-long-jump zone!!!
5025	*/
5026	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5027	{
5028	int rc = VINF_SUCCESS;
5029	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5030	{
5031	uint32_t u32Val = 0;
5032	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5033	pMixedCtx->SysEnter.cs = u32Val;
5034	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5035	}
5036
5037	RTGCUINTREG uGCVal = 0;
5038	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5039	{
5040	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5041	pMixedCtx->SysEnter.eip = uGCVal;
5042	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5043	}
5044	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5045	{
5046	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5047	pMixedCtx->SysEnter.esp = uGCVal;
5048	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5049	}
5050	return rc;
5051	}
5052
5053
5054	/**
5055	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5056	* context.
5057	*
5058	* @returns VBox status code.
5059	* @param pVCpu Pointer to the VMCPU.
5060	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5061	* out-of-sync. Make sure to update the required fields
5062	* before using them.
5063	*
5064	* @remarks No-long-jump zone!!!
5065	*/
5066	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5067	{
5068	RTGCUINTREG uVal = 0;
5069	int rc = VINF_SUCCESS;
5070	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5071	{
5072	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5073	pMixedCtx->fs.u64Base = uVal;
5074	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5075	}
5076	return rc;
5077	}
5078
5079
5080	/**
5081	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5082	* context.
5083	*
5084	* @returns VBox status code.
5085	* @param pVCpu Pointer to the VMCPU.
5086	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5087	* out-of-sync. Make sure to update the required fields
5088	* before using them.
5089	*
5090	* @remarks No-long-jump zone!!!
5091	*/
5092	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5093	{
5094	RTGCUINTREG uVal = 0;
5095	int rc = VINF_SUCCESS;
5096	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5097	{
5098	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5099	pMixedCtx->gs.u64Base = uVal;
5100	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5101	}
5102	return rc;
5103	}
5104
5105
5106	/**
5107	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5108	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5109	*
5110	* @returns VBox status code.
5111	* @param pVCpu Pointer to the VMCPU.
5112	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5113	* out-of-sync. Make sure to update the required fields
5114	* before using them.
5115	*
5116	* @remarks No-long-jump zone!!!
5117	*/
5118	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5119	{
5120	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5121	return VINF_SUCCESS;
5122
5123	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5124	{
5125	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5126	pMsr += i;
5127	switch (pMsr->u32IndexMSR)
5128	{
5129	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5130	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5131	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5132	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5133	#if 0
5134	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5135	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5136	#endif
5137	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5138	default:
5139	{
5140	AssertFailed();
5141	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5142	}
5143	}
5144	}
5145	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5146	return VINF_SUCCESS;
5147	}
5148
5149
5150	/**
5151	* Saves the guest control registers from the current VMCS into the guest-CPU
5152	* context.
5153	*
5154	* @returns VBox status code.
5155	* @param pVCpu Pointer to the VMCPU.
5156	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5157	* out-of-sync. Make sure to update the required fields
5158	* before using them.
5159	*
5160	* @remarks No-long-jump zone!!!
5161	*/
5162	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5163	{
5164	RTGCUINTREG uVal = 0;
5165	RTGCUINTREG uShadow = 0;
5166	int rc = VINF_SUCCESS;
5167
5168	/* Guest CR0. Guest FPU. */
5169	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5170
5171	/* Guest CR4. */
5172	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5173	AssertRCReturn(rc, rc);
5174
5175	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5176	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5177	{
5178	PVM pVM = pVCpu->CTX_SUFF(pVM);
5179	if ( pVM->hm.s.fNestedPaging
5180	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5181	{
5182	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5183	if (pMixedCtx->cr3 != uVal)
5184	{
5185	CPUMSetGuestCR3(pVCpu, uVal);
5186	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5187	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5188	}
5189
5190	/* We require EFER to check PAE mode. */
5191	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5192
5193	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5194	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5195	{
5196	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5197	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5198	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5199	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5200	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5201	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5202	}
5203	AssertRCReturn(rc, rc);
5204	}
5205
5206	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5207	}
5208	return rc;
5209	}
5210
5211
5212	/**
5213	* Reads a guest segment register from the current VMCS into the guest-CPU
5214	* context.
5215	*
5216	* @returns VBox status code.
5217	* @param pVCpu Pointer to the VMCPU.
5218	* @param idxSel Index of the selector in the VMCS.
5219	* @param idxLimit Index of the segment limit in the VMCS.
5220	* @param idxBase Index of the segment base in the VMCS.
5221	* @param idxAccess Index of the access rights of the segment in the VMCS.
5222	* @param pSelReg Pointer to the segment selector.
5223	*
5224	* @remarks No-long-jump zone!!!
5225	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5226	* as that takes care of whether to read from the VMCS cache or not.
5227	*/
5228	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5229	PCPUMSELREG pSelReg)
5230	{
5231	uint32_t u32Val = 0;
5232	int rc = VMXReadVmcs32(idxSel, &u32Val);
5233	pSelReg->Sel = (uint16_t)u32Val;
5234	pSelReg->ValidSel = (uint16_t)u32Val;
5235	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5236
5237	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5238	pSelReg->u32Limit = u32Val;
5239
5240	RTGCUINTREG uGCVal = 0;
5241	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5242	pSelReg->u64Base = uGCVal;
5243
5244	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5245	pSelReg->Attr.u = u32Val;
5246	AssertRCReturn(rc, rc);
5247
5248	/*
5249	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5250	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5251	*/
5252	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5253	{
5254	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5255	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5256	}
5257	return rc;
5258	}
5259
5260
5261	/**
5262	* Saves the guest segment registers from the current VMCS into the guest-CPU
5263	* context.
5264	*
5265	* @returns VBox status code.
5266	* @param pVCpu Pointer to the VMCPU.
5267	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5268	* out-of-sync. Make sure to update the required fields
5269	* before using them.
5270	*
5271	* @remarks No-long-jump zone!!!
5272	*/
5273	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5274	{
5275	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5276	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5277	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5278	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5279	#else
5280	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5281	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5282	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5283	#endif
5284
5285	int rc = VINF_SUCCESS;
5286
5287	/* Guest segment registers. */
5288	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5289	{
5290	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5291	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5292	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5293	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5294	rc \|= VMXLOCAL_READ_SEG(ES, es);
5295	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5296	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5297	AssertRCReturn(rc, rc);
5298
5299	/* Restore segment attributes for real-on-v86 mode hack. */
5300	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5301	{
5302	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5303	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5304	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5305	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5306	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5307	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5308	}
5309	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5310	}
5311
5312	/* Guest LDTR. */
5313	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5314	{
5315	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5316	AssertRCReturn(rc, rc);
5317	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5318	}
5319
5320	/* Guest GDTR. */
5321	RTGCUINTREG uGCVal = 0;
5322	uint32_t u32Val = 0;
5323	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5324	{
5325	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5326	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5327	pMixedCtx->gdtr.pGdt = uGCVal;
5328	pMixedCtx->gdtr.cbGdt = u32Val;
5329	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5330	}
5331
5332	/* Guest IDTR. */
5333	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5334	{
5335	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5336	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5337	pMixedCtx->idtr.pIdt = uGCVal;
5338	pMixedCtx->idtr.cbIdt = u32Val;
5339	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5340	}
5341
5342	/* Guest TR. */
5343	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5344	{
5345	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5346
5347	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5348	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5349	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5350	AssertRCReturn(rc, rc);
5351	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5352	}
5353	return rc;
5354	}
5355
5356
5357	/**
5358	* Saves the guest debug registers from the current VMCS into the guest-CPU
5359	* context.
5360	*
5361	* @returns VBox status code.
5362	* @param pVCpu Pointer to the VMCPU.
5363	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5364	* out-of-sync. Make sure to update the required fields
5365	* before using them.
5366	*
5367	* @remarks No-long-jump zone!!!
5368	*/
5369	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5370	{
5371	int rc = VINF_SUCCESS;
5372	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5373	{
5374	RTGCUINTREG uVal;
5375	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5376	pMixedCtx->dr[7] = uVal;
5377
5378	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5379	}
5380	return rc;
5381	}
5382
5383
5384	/**
5385	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5386	*
5387	* @returns VBox status code.
5388	* @param pVCpu Pointer to the VMCPU.
5389	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5390	* out-of-sync. Make sure to update the required fields
5391	* before using them.
5392	*
5393	* @remarks No-long-jump zone!!!
5394	*/
5395	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5396	{
5397	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5398	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5399	return VINF_SUCCESS;
5400	}
5401
5402
5403	/**
5404	* Saves the entire guest state from the currently active VMCS into the
5405	* guest-CPU context. This essentially VMREADs all guest-data.
5406	*
5407	* @returns VBox status code.
5408	* @param pVCpu Pointer to the VMCPU.
5409	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5410	* out-of-sync. Make sure to update the required fields
5411	* before using them.
5412	*/
5413	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5414	{
5415	Assert(pVCpu);
5416	Assert(pMixedCtx);
5417
5418	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5419	return VINF_SUCCESS;
5420
5421	VMMRZCallRing3Disable(pVCpu);
5422	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5423	LogFunc(("\n"));
5424
5425	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5427
5428	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5429	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5430
5431	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5433
5434	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5435	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5436
5437	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5438	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5439
5440	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5441	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5442
5443	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5444	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5445
5446	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5447	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5448
5449	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5451
5452	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5454
5455	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5456	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5457
5458	VMMRZCallRing3Enable(pVCpu);
5459	return rc;
5460	}
5461
5462
5463	/**
5464	* Check per-VM and per-VCPU force flag actions that require us to go back to
5465	* ring-3 for one reason or another.
5466	*
5467	* @returns VBox status code (information status code included).
5468	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5469	* ring-3.
5470	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5471	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5472	* interrupts)
5473	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5474	* all EMTs to be in ring-3.
5475	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5476	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5477	* to the EM loop.
5478	*
5479	* @param pVM Pointer to the VM.
5480	* @param pVCpu Pointer to the VMCPU.
5481	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5482	* out-of-sync. Make sure to update the required fields
5483	* before using them.
5484	*/
5485	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5486	{
5487	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5488
5489	int rc = VERR_INTERNAL_ERROR_5;
5490	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5491	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5492	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5493	{
5494	/* We need the control registers now, make sure the guest-CPU context is updated. */
5495	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5496	AssertRCReturn(rc, rc);
5497
5498	/* Pending HM CR3 sync. */
5499	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5500	{
5501	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5502	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5503	}
5504	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5505	{
5506	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5507	AssertRC(rc);
5508	}
5509
5510	/* Pending PGM C3 sync. */
5511	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5512	{
5513	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5514	if (rc != VINF_SUCCESS)
5515	{
5516	AssertRC(rc);
5517	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5518	return rc;
5519	}
5520	}
5521
5522	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5523	/* -XXX- what was that about single stepping? */
5524	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5525	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5526	{
5527	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5528	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5529	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5530	return rc;
5531	}
5532
5533	/* Pending VM request packets, such as hardware interrupts. */
5534	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5535	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5536	{
5537	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5538	return VINF_EM_PENDING_REQUEST;
5539	}
5540
5541	/* Pending PGM pool flushes. */
5542	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5543	{
5544	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5545	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5546	}
5547
5548	/* Pending DMA requests. */
5549	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5550	{
5551	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5552	return VINF_EM_RAW_TO_R3;
5553	}
5554	}
5555
5556	/* Paranoia. */
5557	Assert(rc != VERR_EM_INTERPRETER);
5558	return VINF_SUCCESS;
5559	}
5560
5561
5562	/**
5563	* Converts any TRPM trap into a pending VMX event. This is typically used when
5564	* entering from ring-3 (not longjmp returns).
5565	*
5566	* @param pVCpu Pointer to the VMCPU.
5567	* @param pCtx Pointer to the guest-CPU context.
5568	*/
5569	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5570	{
5571	if (!TRPMHasTrap(pVCpu))
5572	{
5573	Assert(!pVCpu->hm.s.Event.fPending);
5574	return;
5575	}
5576
5577	uint8_t uVector;
5578	TRPMEVENT enmTrpmEvent;
5579	RTGCUINT uErrCode;
5580	RTGCUINTPTR GCPtrFaultAddress;
5581	uint8_t cbInstr;
5582
5583	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5584	AssertRC(rc);
5585
5586	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5587	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5588	uint32_t u32ErrCode = uErrCode;
5589	if (enmTrpmEvent == TRPM_TRAP)
5590	{
5591	switch (uVector)
5592	{
5593	case X86_XCPT_BP:
5594	case X86_XCPT_OF:
5595	{
5596	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5597	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5598	break;
5599	}
5600
5601	case X86_XCPT_PF:
5602	pCtx->cr2 = GCPtrFaultAddress;
5603	/* no break */
5604	case X86_XCPT_DF:
5605	case X86_XCPT_TS:
5606	case X86_XCPT_NP:
5607	case X86_XCPT_SS:
5608	case X86_XCPT_GP:
5609	case X86_XCPT_AC:
5610	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5611	them which VT-x/VMM pushes to the guest stack. */
5612	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5613	/* no break! */
5614	default:
5615	{
5616	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5617	break;
5618	}
5619	}
5620	}
5621	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5622	{
5623	if (uVector != X86_XCPT_NMI)
5624	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5625	else
5626	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5627	}
5628	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5629	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5630	else
5631	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5632
5633	rc = TRPMResetTrap(pVCpu);
5634	AssertRC(rc);
5635	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5636	u32IntrInfo, enmTrpmEvent, cbInstr, u32ErrCode, GCPtrFaultAddress));
5637	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5638	}
5639
5640
5641	/**
5642	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5643	* VT-x to execute any instruction.
5644	*
5645	* @param pvCpu Pointer to the VMCPU.
5646	*/
5647	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5648	{
5649	if (pVCpu->hm.s.Event.fPending)
5650	{
5651	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5652	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5653	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5654	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5655
5656	/* If a trap was already pending, we did something wrong! */
5657	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5658
5659	TRPMEVENT enmTrapType;
5660	switch (uVectorType)
5661	{
5662	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5663	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5664	enmTrapType = TRPM_HARDWARE_INT;
5665	break;
5666	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5667	enmTrapType = TRPM_SOFTWARE_INT;
5668	break;
5669	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5670	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5671	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5672	enmTrapType = TRPM_TRAP;
5673	break;
5674	default:
5675	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5676	enmTrapType = TRPM_32BIT_HACK;
5677	break;
5678	}
5679
5680	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5681	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5682	AssertRC(rc);
5683
5684	if (fErrorCodeValid)
5685	TRPMSetErrorCode(pVCpu, uErrorCode);
5686	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5687	&& uVector == X86_XCPT_PF)
5688	{
5689	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5690	}
5691	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5692	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5693	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5694	{
5695	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5696	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5697	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5698	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5699	}
5700	pVCpu->hm.s.Event.fPending = false;
5701	}
5702	}
5703
5704
5705	/**
5706	* Does the necessary state syncing before doing a longjmp to ring-3.
5707	*
5708	* @param pVM Pointer to the VM.
5709	* @param pVCpu Pointer to the VMCPU.
5710	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5711	* out-of-sync. Make sure to update the required fields
5712	* before using them.
5713	* @param rcExit The reason for exiting to ring-3. Can be
5714	* VINF_VMM_UNKNOWN_RING3_CALL.
5715	*
5716	* @remarks No-long-jmp zone!!!
5717	*/
5718	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5719	{
5720	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5721	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5722
5723	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5724	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5725	AssertRC(rc);
5726
5727	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5728	if (CPUMIsGuestFPUStateActive(pVCpu))
5729	{
5730	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5731	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5732	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5733	}
5734
5735	/* Restore debug registers if necessary and resync on next R0 reentry. */
5736	if (CPUMIsGuestDebugStateActive(pVCpu))
5737	{
5738	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5739	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5740	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5741	}
5742	else if (CPUMIsHyperDebugStateActive(pVCpu))
5743	{
5744	CPUMR0LoadHostDebugState(pVM, pVCpu);
5745	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5746	}
5747
5748	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5749	}
5750
5751
5752	/**
5753	* An action requires us to go back to ring-3. This function does the necessary
5754	* steps before we can safely return to ring-3. This is not the same as longjmps
5755	* to ring-3, this is voluntary.
5756	*
5757	* @param pVM Pointer to the VM.
5758	* @param pVCpu Pointer to the VMCPU.
5759	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5760	* out-of-sync. Make sure to update the required fields
5761	* before using them.
5762	* @param rcExit The reason for exiting to ring-3. Can be
5763	* VINF_VMM_UNKNOWN_RING3_CALL.
5764	*/
5765	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5766	{
5767	Assert(pVM);
5768	Assert(pVCpu);
5769	Assert(pMixedCtx);
5770	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5771
5772	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5773	{
5774	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5775	return;
5776	}
5777	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5778	{
5779	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5780	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5781	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5782	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5783	return;
5784	}
5785
5786	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5787	VMMRZCallRing3Disable(pVCpu);
5788	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5789
5790	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5791	hmR0VmxUpdateTRPM(pVCpu);
5792
5793	/* Sync. the guest state. */
5794	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5795	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5796
5797	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5798	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5799	\| CPUM_CHANGED_LDTR
5800	\| CPUM_CHANGED_GDTR
5801	\| CPUM_CHANGED_IDTR
5802	\| CPUM_CHANGED_TR
5803	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5804
5805	/* On our way back from ring-3 the following needs to be done. */
5806	/** @todo This can change with preemption hooks. */
5807	if (rcExit == VINF_EM_RAW_INTERRUPT)
5808	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5809	else
5810	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5811
5812	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5813	VMMRZCallRing3Enable(pVCpu);
5814	}
5815
5816
5817	/**
5818	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5819	* longjump to ring-3 and possibly get preempted.
5820	*
5821	* @param pVCpu Pointer to the VMCPU.
5822	* @param enmOperation The operation causing the ring-3 longjump.
5823	* @param pvUser The user argument (pointer to the possibly
5824	* out-of-date guest-CPU context).
5825	*
5826	* @remarks Must never be called with @a enmOperation ==
5827	* VMMCALLRING3_VM_R0_ASSERTION.
5828	*/
5829	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5830	{
5831	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5832	Assert(pVCpu);
5833	Assert(pvUser);
5834	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5835	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5836
5837	VMMRZCallRing3Disable(pVCpu);
5838	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5839	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5840	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5841	VMMRZCallRing3Enable(pVCpu);
5842	}
5843
5844
5845	/**
5846	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5847	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5848	*
5849	* @returns VBox status code.
5850	* @param pVCpu Pointer to the VMCPU.
5851	*/
5852	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5853	{
5854	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5855	{
5856	/* Enable interrupt-window exiting if it's not in effect already. */
5857	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5858	{
5859	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5860	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5861	AssertRC(rc);
5862	}
5863	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5864	}
5865
5866
5867	/**
5868	* Injects any pending events into the guest if the guest is in a state to
5869	* receive them.
5870	*
5871	* @returns VBox status code (informational status codes included).
5872	* @param pVCpu Pointer to the VMCPU.
5873	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5874	* out-of-sync. Make sure to update the required fields
5875	* before using them.
5876	*/
5877	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5878	{
5879	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5880	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5881	const bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5882	const bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5883
5884	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5885	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5886	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5887	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5888
5889	int rc = VINF_SUCCESS;
5890	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5891	{
5892	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5893	bool fInject = true;
5894	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5895	{
5896	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5897	AssertRCReturn(rc, rc);
5898	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5899	if ( fBlockInt
5900	\|\| fBlockSti
5901	\|\| fBlockMovSS)
5902	{
5903	fInject = false;
5904	}
5905	}
5906	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5907	&& ( fBlockMovSS
5908	\|\| fBlockSti))
5909	{
5910	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5911	fInject = false;
5912	}
5913
5914	if (fInject)
5915	{
5916	Log(("Injecting pending event\n"));
5917	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5918	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5919	AssertRCReturn(rc, rc);
5920	pVCpu->hm.s.Event.fPending = false;
5921	}
5922	else
5923	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5924	} /** @todo SMI. SMIs take priority over NMIs. */
5925	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5926	{
5927	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5928	if ( !fBlockMovSS
5929	&& !fBlockSti)
5930	{
5931	Log(("Injecting NMI\n"));
5932	RTGCUINTPTR uIntrInfo;
5933	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5934	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5935	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5936	AssertRCReturn(rc, rc);
5937	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5938	}
5939	else
5940	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5941	}
5942	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5943	{
5944	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5945	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5946	AssertRCReturn(rc, rc);
5947	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5948	if ( !fBlockInt
5949	&& !fBlockSti
5950	&& !fBlockMovSS)
5951	{
5952	uint8_t u8Interrupt;
5953	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5954	if (RT_SUCCESS(rc))
5955	{
5956	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5957	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5958	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5959	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5960	}
5961	else
5962	{
5963	/** @todo Does this actually happen? If not turn it into an assertion. */
5964	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5965	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5966	rc = VINF_SUCCESS;
5967	}
5968	}
5969	else
5970	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5971	}
5972
5973	/*
5974	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5975	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5976	*/
5977	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5978	AssertRC(rc2);
5979	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5980	return rc;
5981	}
5982
5983
5984	/**
5985	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5986	*
5987	* @param pVCpu Pointer to the VMCPU.
5988	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5989	* out-of-sync. Make sure to update the required fields
5990	* before using them.
5991	*/
5992	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5993	{
5994	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5995	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5996	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5997	}
5998
5999
6000	/**
6001	* Injects a double-fault (#DF) exception into the VM.
6002	*
6003	* @returns VBox status code (informational status code included).
6004	* @param pVCpu Pointer to the VMCPU.
6005	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6006	* out-of-sync. Make sure to update the required fields
6007	* before using them.
6008	*/
6009	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6010	{
6011	/* Inject the double-fault. */
6012	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6013	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6014	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6015	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6016	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6017	}
6018
6019
6020	/**
6021	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6022	*
6023	* @param pVCpu Pointer to the VMCPU.
6024	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6025	* out-of-sync. Make sure to update the required fields
6026	* before using them.
6027	*/
6028	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6029	{
6030	/* Inject the debug-exception. */
6031	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6032	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6033	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6034	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6035	}
6036
6037
6038	/**
6039	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6040	*
6041	* @param pVCpu Pointer to the VMCPU.
6042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6043	* out-of-sync. Make sure to update the required fields
6044	* before using them.
6045	* @param cbInstr The value of RIP that is to be pushed on the guest
6046	* stack.
6047	*/
6048	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6049	{
6050	/* Inject the overflow exception. */
6051	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6052	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6053	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6054	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6055	}
6056
6057
6058	/**
6059	* Injects a general-protection (#GP) fault into the VM.
6060	*
6061	* @returns VBox status code (informational status code included).
6062	* @param pVCpu Pointer to the VMCPU.
6063	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6064	* out-of-sync. Make sure to update the required fields
6065	* before using them.
6066	* @param u32ErrorCode The error code associated with the #GP.
6067	*/
6068	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6069	uint32_t *puIntrState)
6070	{
6071	/* Inject the general-protection fault. */
6072	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6073	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6074	if (fErrorCodeValid)
6075	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6076	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6077	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6078	}
6079
6080
6081	/**
6082	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6083	*
6084	* @param pVCpu Pointer to the VMCPU.
6085	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6086	* out-of-sync. Make sure to update the required fields
6087	* before using them.
6088	* @param uVector The software interrupt vector number.
6089	* @param cbInstr The value of RIP that is to be pushed on the guest
6090	* stack.
6091	*/
6092	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6093	{
6094	/* Inject the INTn. */
6095	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6096	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6097	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6098	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6099	}
6100
6101
6102	/**
6103	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6104	* stack.
6105	*
6106	* @returns VBox status code (information status code included).
6107	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6108	* @param pVM Pointer to the VM.
6109	* @param pMixedCtx Pointer to the guest-CPU context.
6110	* @param uValue The value to push to the guest stack.
6111	*/
6112	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6113	{
6114	/*
6115	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6116	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6117	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6118	*/
6119	if (pMixedCtx->sp == 1)
6120	return VINF_EM_RESET;
6121	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6122	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6123	AssertRCReturn(rc, rc);
6124	return rc;
6125	}
6126
6127
6128	/**
6129	* Injects an event into the guest upon VM-entry by updating the relevant fields
6130	* in the VM-entry area in the VMCS.
6131	*
6132	* @returns VBox status code (informational error codes included).
6133	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6134	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6135	*
6136	* @param pVCpu Pointer to the VMCPU.
6137	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6138	* out-of-sync. Make sure to update the required fields
6139	* before using them.
6140	* @param u64IntrInfo The VM-entry interruption-information field.
6141	* @param cbInstr The VM-entry instruction length in bytes (for software
6142	* interrupts, exceptions and privileged software
6143	* exceptions).
6144	* @param u32ErrCode The VM-entry exception error code.
6145	* @param puIntrState Pointer to the current guest interruptibility-state.
6146	* This interruptibility-state will be updated if
6147	* necessary. This cannot not be NULL.
6148	*
6149	* @remarks No-long-jump zone!!!
6150	* @remarks Requires CR0!
6151	*/
6152	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6153	uint32_t u32ErrCode, uint32_t *puIntrState)
6154	{
6155	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6156	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6157	Assert(puIntrState);
6158	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6159
6160	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6161	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6162
6163	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6164	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6165	\|\| !((*puIntrState) & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6166
6167	/* We require CR0 to check if the guest is in real-mode. */
6168	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6169	AssertRCReturn(rc, rc);
6170
6171	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6172
6173	/*
6174	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6175	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6176	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6177	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6178	*/
6179	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6180	{
6181	PVM pVM = pVCpu->CTX_SUFF(pVM);
6182	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6183	{
6184	Assert(PDMVmmDevHeapIsEnabled(pVM));
6185	Assert(pVM->hm.s.vmx.pRealModeTSS);
6186
6187	/* Save the required guest state bits from the VMCS. */
6188	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6189	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6190	AssertRCReturn(rc, rc);
6191
6192	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6193	const size_t cbIdtEntry = 4;
6194	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6195	{
6196	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6197	if (uVector == X86_XCPT_DF)
6198	return VINF_EM_RESET;
6199	else if (uVector == X86_XCPT_GP)
6200	{
6201	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6202	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6203	}
6204
6205	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6206	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6207	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6208	}
6209
6210	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6211	uint16_t uGuestIp = pMixedCtx->ip;
6212	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6213	{
6214	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6215	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6216	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6217	}
6218	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6219	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6220
6221	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6222	uint16_t offIdtEntry = 0;
6223	RTSEL selIdtEntry = 0;
6224	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6225	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6226	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6227	AssertRCReturn(rc, rc);
6228
6229	/* Construct the stack frame for the interrupt/exception handler. */
6230	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6231	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6232	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6233	AssertRCReturn(rc, rc);
6234
6235	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6236	if (rc == VINF_SUCCESS)
6237	{
6238	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6239	pMixedCtx->rip = offIdtEntry;
6240	pMixedCtx->cs.Sel = selIdtEntry;
6241	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6242	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6243	\| HM_CHANGED_GUEST_RIP
6244	\| HM_CHANGED_GUEST_RFLAGS
6245	\| HM_CHANGED_GUEST_RSP;
6246
6247	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6248	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6249	{
6250	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6251	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6252	Log(("Clearing inhibition due to STI.\n"));
6253	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6254	}
6255	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6256	}
6257	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6258	return rc;
6259	}
6260	else
6261	{
6262	/*
6263	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6264	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6265	*/
6266	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6267	}
6268	}
6269
6270	/* Validate. */
6271	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6272	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6273	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6274
6275	/* Inject. */
6276	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6277	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6278	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6279	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6280	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6281	AssertRCReturn(rc, rc);
6282	return rc;
6283	}
6284
6285
6286	/**
6287	* Enters the VT-x session.
6288	*
6289	* @returns VBox status code.
6290	* @param pVM Pointer to the VM.
6291	* @param pVCpu Pointer to the VMCPU.
6292	* @param pCpu Pointer to the CPU info struct.
6293	*/
6294	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6295	{
6296	AssertPtr(pVM);
6297	AssertPtr(pVCpu);
6298	Assert(pVM->hm.s.vmx.fSupported);
6299	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6300	NOREF(pCpu);
6301
6302	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6303
6304	/* Make sure we're in VMX root mode. */
6305	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6306	if (!(u32HostCR4 & X86_CR4_VMXE))
6307	{
6308	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6309	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6310	}
6311
6312	/* Load the active VMCS as the current one. */
6313	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6314	if (RT_FAILURE(rc))
6315	return rc;
6316
6317	/** @todo this will change with preemption hooks where can can VMRESUME as long
6318	* as we're no preempted. */
6319	pVCpu->hm.s.fResumeVM = false;
6320	return VINF_SUCCESS;
6321	}
6322
6323
6324	/**
6325	* Leaves the VT-x session.
6326	*
6327	* @returns VBox status code.
6328	* @param pVM Pointer to the VM.
6329	* @param pVCpu Pointer to the VMCPU.
6330	* @param pCtx Pointer to the guest-CPU context.
6331	*/
6332	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6333	{
6334	AssertPtr(pVCpu);
6335	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6336	NOREF(pVM);
6337	NOREF(pCtx);
6338
6339	/** @todo this will change with preemption hooks where we only VMCLEAR when
6340	* we are actually going to be preempted, not all the time like we
6341	* currently do. */
6342	/*
6343	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6344	* and mark the VMCS launch-state as "clear".
6345	*/
6346	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6347	return rc;
6348	}
6349
6350
6351	/**
6352	* Saves the host state in the VMCS host-state.
6353	* Sets up the VM-exit MSR-load area.
6354	*
6355	* The CPU state will be loaded from these fields on every successful VM-exit.
6356	*
6357	* @returns VBox status code.
6358	* @param pVM Pointer to the VM.
6359	* @param pVCpu Pointer to the VMCPU.
6360	*
6361	* @remarks No-long-jump zone!!!
6362	*/
6363	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6364	{
6365	AssertPtr(pVM);
6366	AssertPtr(pVCpu);
6367	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6368
6369	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6370
6371	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6372	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6373	return VINF_SUCCESS;
6374
6375	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6377
6378	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6380
6381	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6383
6384	/* Reset the host-state-changed flag. */
6385	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6386	return rc;
6387	}
6388
6389
6390	/**
6391	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6392	* loaded from these fields on every successful VM-entry.
6393	*
6394	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6395	* Sets up the VM-entry controls.
6396	* Sets up the appropriate VMX non-root function to execute guest code based on
6397	* the guest CPU mode.
6398	*
6399	* @returns VBox status code.
6400	* @param pVM Pointer to the VM.
6401	* @param pVCpu Pointer to the VMCPU.
6402	* @param pCtx Pointer to the guest-CPU context.
6403	*
6404	* @remarks No-long-jump zone!!!
6405	*/
6406	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6407	{
6408	AssertPtr(pVM);
6409	AssertPtr(pVCpu);
6410	AssertPtr(pCtx);
6411	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6412
6413	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6414
6415	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6416
6417	/* Determine real-on-v86 mode. */
6418	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6419	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6420	&& CPUMIsGuestInRealModeEx(pCtx))
6421	{
6422	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6423	}
6424
6425	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6427
6428	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6429	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6430
6431	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6432	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6433
6434	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6435	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6436
6437	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6438	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6439
6440	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6441	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6442
6443	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6444	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6445
6446	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6447	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6448
6449	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6451
6452	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6454
6455	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6456	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6457	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6458
6459	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6460	return rc;
6461	}
6462
6463
6464	/**
6465	* Does the preparations before executing guest code in VT-x.
6466	*
6467	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6468	* recompiler. We must be cautious what we do here regarding committing
6469	* guest-state information into the the VMCS assuming we assuredly execute the
6470	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6471	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6472	* that the recompiler can (and should) use them when it resumes guest
6473	* execution. Otherwise such operations must be done when we can no longer
6474	* exit to ring-3.
6475	*
6476	* @returns VBox status code (informational status codes included).
6477	* @retval VINF_SUCCESS if we can proceed with running the guest.
6478	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6479	* into the guest.
6480	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6481	*
6482	* @param pVM Pointer to the VM.
6483	* @param pVCpu Pointer to the VMCPU.
6484	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6485	* out-of-sync. Make sure to update the required fields
6486	* before using them.
6487	* @param pVmxTransient Pointer to the VMX transient structure.
6488	*
6489	* @remarks Called with preemption disabled.
6490	*/
6491	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6492	{
6493	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6494
6495	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6496	PGMRZDynMapFlushAutoSet(pVCpu);
6497	#endif
6498
6499	/* Check force flag actions that might require us to go back to ring-3. */
6500	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6501	if (rc != VINF_SUCCESS)
6502	return rc;
6503
6504	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6505	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6506	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6507	{
6508	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6509	RTGCPHYS GCPhysApicBase;
6510	GCPhysApicBase = pMixedCtx->msrApicBase;
6511	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6512
6513	/* Unalias any existing mapping. */
6514	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6515	AssertRCReturn(rc, rc);
6516
6517	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6518	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6519	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6520	AssertRCReturn(rc, rc);
6521
6522	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6523	}
6524
6525	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6526	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6527	pVmxTransient->uEFlags = ASMIntDisableFlags();
6528	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6529	{
6530	ASMSetFlags(pVmxTransient->uEFlags);
6531	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6532	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6533	return VINF_EM_RAW_INTERRUPT;
6534	}
6535	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6536	#endif
6537
6538	/*
6539	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6540	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6541	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6542	*/
6543	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6544	AssertRCReturn(rc, rc);
6545	return rc;
6546	}
6547
6548
6549	/**
6550	* Prepares to run guest code in VT-x and we've committed to doing so. This
6551	* means there is no backing out to ring-3 or anywhere else at this
6552	* point.
6553	*
6554	* @param pVM Pointer to the VM.
6555	* @param pVCpu Pointer to the VMCPU.
6556	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6557	* out-of-sync. Make sure to update the required fields
6558	* before using them.
6559	* @param pVmxTransient Pointer to the VMX transient structure.
6560	*
6561	* @remarks Called with preemption disabled.
6562	* @remarks No-long-jump zone!!!
6563	*/
6564	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6565	{
6566	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6567	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6568
6569	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6570	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6571	pVmxTransient->uEFlags = ASMIntDisableFlags();
6572	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6573	#endif
6574
6575	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6576	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6577	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6578	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6579	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6580	#endif
6581	int rc = VINF_SUCCESS;
6582	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6583	{
6584	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6585	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6586	}
6587	else if (pVCpu->hm.s.fContextUseFlags)
6588	{
6589	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6590	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6591	}
6592	AssertRC(rc);
6593	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6594
6595	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6596	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6597	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6598
6599	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6600	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6601
6602	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6603	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6604	{
6605	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6606	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6607	}
6608
6609	/*
6610	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6611	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6612	*/
6613	if (pVM->hm.s.fTPRPatchingActive)
6614	{
6615	Assert(!CPUMIsGuestInLongMode(pVCpu));
6616
6617	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6618	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6619	AssertRC(rc);
6620
6621	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6622	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6623	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6624	see hmR0VmxLoadGuestApicState(). */
6625	}
6626
6627	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6628	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6629	to start executing. */
6630	}
6631
6632
6633	/**
6634	* Performs some essential restoration of state after running guest code in
6635	* VT-x.
6636	*
6637	* @param pVM Pointer to the VM.
6638	* @param pVCpu Pointer to the VMCPU.
6639	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6640	* out-of-sync. Make sure to update the required fields
6641	* before using them.
6642	* @param pVmxTransient Pointer to the VMX transient structure.
6643	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6644	*
6645	* @remarks Called with interrupts disabled.
6646	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6647	* unconditionally when it is safe to do so.
6648	*/
6649	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6650	{
6651	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6652	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6653
6654	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6655	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6656	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6657	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6658	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6659
6660	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6661	{
6662	/** @todo Find a way to fix hardcoding a guestimate. */
6663	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6664	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6665	}
6666
6667	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6668	Assert(!(ASMGetFlags() & X86_EFL_IF));
6669	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6670
6671	/* Restore the effects of TPR patching if any. */
6672	if (pVM->hm.s.fTPRPatchingActive)
6673	{
6674	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6675	AssertRC(rc);
6676	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6677	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6678	}
6679
6680	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6681	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6682
6683	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6684	uint32_t uExitReason;
6685	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6686	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6687	AssertRC(rc);
6688	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6689	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6690
6691	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6692	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6693
6694	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6695	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6696	{
6697	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6698	return;
6699	}
6700
6701	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6702	{
6703	/* Update the guest interruptibility-state from the VMCS. */
6704	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6705	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6706	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6707	AssertRC(rc);
6708	#endif
6709	/*
6710	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6711	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6712	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6713	*/
6714	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6715	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6716	{
6717	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6718	AssertRC(rc);
6719	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6720	}
6721	}
6722	}
6723
6724
6725	/**
6726	* Runs the guest code using VT-x.
6727	*
6728	* @returns VBox status code.
6729	* @param pVM Pointer to the VM.
6730	* @param pVCpu Pointer to the VMCPU.
6731	* @param pCtx Pointer to the guest-CPU context.
6732	*
6733	* @remarks Called with preemption disabled.
6734	*/
6735	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6736	{
6737	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6738	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6739
6740	VMXTRANSIENT VmxTransient;
6741	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6742	int rc = VERR_INTERNAL_ERROR_5;
6743	uint32_t cLoops = 0;
6744	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6745
6746	for (;; cLoops++)
6747	{
6748	Assert(!HMR0SuspendPending());
6749	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6750	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6751	(unsigned)RTMpCpuId(), cLoops));
6752
6753	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6754	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6755	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6756	if (rc != VINF_SUCCESS)
6757	break;
6758
6759	/*
6760	* No longjmps to ring-3 from this point on!!!
6761	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6762	* This also disables flushing of the R0-logger instance (if any).
6763	*/
6764	VMMRZCallRing3Disable(pVCpu);
6765	VMMRZCallRing3RemoveNotification(pVCpu);
6766	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6767
6768	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6769	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6770
6771	/*
6772	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6773	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6774	*/
6775	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6776	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6777	{
6778	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6779	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6780	return rc;
6781	}
6782
6783	/* Handle the VM-exit. */
6784	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6785	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6786	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6787	#ifdef HM_PROFILE_EXIT_DISPATCH
6788	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6789	#endif
6790	#ifdef HMVMX_USE_FUNCTION_TABLE
6791	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6792	#else
6793	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6794	#endif
6795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6796	if (rc != VINF_SUCCESS)
6797	break;
6798	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6799	{
6800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6801	rc = VINF_EM_RAW_INTERRUPT;
6802	break;
6803	}
6804	}
6805
6806	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6807	if (rc == VERR_EM_INTERPRETER)
6808	rc = VINF_EM_RAW_EMULATE_INSTR;
6809	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6810	return rc;
6811	}
6812
6813
6814	#ifndef HMVMX_USE_FUNCTION_TABLE
6815	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6816	{
6817	int rc;
6818	switch (rcReason)
6819	{
6820	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6853
6854	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6856	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6857	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6862	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6863
6864	case VMX_EXIT_VMCALL:
6865	case VMX_EXIT_VMCLEAR:
6866	case VMX_EXIT_VMLAUNCH:
6867	case VMX_EXIT_VMPTRLD:
6868	case VMX_EXIT_VMPTRST:
6869	case VMX_EXIT_VMREAD:
6870	case VMX_EXIT_VMRESUME:
6871	case VMX_EXIT_VMWRITE:
6872	case VMX_EXIT_VMXOFF:
6873	case VMX_EXIT_VMXON:
6874	case VMX_EXIT_INVEPT:
6875	case VMX_EXIT_INVVPID:
6876	case VMX_EXIT_VMFUNC:
6877	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6878	break;
6879	default:
6880	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6881	break;
6882	}
6883	return rc;
6884	}
6885	#endif
6886
6887
6888	/** Profiling macro. */
6889	#ifdef HM_PROFILE_EXIT_DISPATCH
6890	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6891	#else
6892	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6893	#endif
6894
6895
6896	#ifdef DEBUG
6897	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6898	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6899	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6900	# define VMX_ASSERT_PREEMPT_CPUID() \
6901	do \
6902	{ \
6903	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6904	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6905	} while (0)
6906
6907	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6908	do { \
6909	AssertPtr(pVCpu); \
6910	AssertPtr(pMixedCtx); \
6911	AssertPtr(pVmxTransient); \
6912	Assert(pVmxTransient->fVMEntryFailed == false); \
6913	Assert(ASMIntAreEnabled()); \
6914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6915	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6916	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6917	(unsigned)pVCpu->idCpu)); \
6918	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6919	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6920	VMX_ASSERT_PREEMPT_CPUID(); \
6921	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6922	} while (0)
6923	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6924	do { \
6925	LogFunc(("\n")); \
6926	} while(0)
6927	#else /* Release builds */
6928	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6929	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6930	#endif
6931
6932
6933	/**
6934	* Advances the guest RIP after reading it from the VMCS.
6935	*
6936	* @returns VBox status code.
6937	* @param pVCpu Pointer to the VMCPU.
6938	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6939	* out-of-sync. Make sure to update the required fields
6940	* before using them.
6941	* @param pVmxTransient Pointer to the VMX transient structure.
6942	*
6943	* @remarks No-long-jump zone!!!
6944	*/
6945	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6946	{
6947	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6948	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6949	AssertRCReturn(rc, rc);
6950
6951	pMixedCtx->rip += pVmxTransient->cbInstr;
6952	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6953	return rc;
6954	}
6955
6956
6957	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6958	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6959	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6960	/**
6961	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6962	*/
6963	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6964	{
6965	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6967	return VINF_SUCCESS;
6968	}
6969
6970
6971	/**
6972	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6973	*/
6974	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6975	{
6976	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6977	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6978
6979	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6980	AssertRCReturn(rc, rc);
6981
6982	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6983	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6984	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6985
6986	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6987	{
6988	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6989	return VINF_EM_RAW_INTERRUPT;
6990	}
6991
6992	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6993	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6994	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6995	{
6996	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6997	return VINF_SUCCESS;
6998	}
6999	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7000	{
7001	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7002	return rc;
7003	}
7004
7005	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7006	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7007	switch (uIntrType)
7008	{
7009	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7010	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7011	/* no break */
7012	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
7013	{
7014	switch (uVector)
7015	{
7016	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7017	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7018	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7019	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7020	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7021	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7022	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7023	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7024	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7025	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7026	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7027	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7028	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7029	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7030	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7031	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7032	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7033	#endif
7034	default:
7035	{
7036	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7037	AssertRCReturn(rc, rc);
7038
7039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7040	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7041	{
7042	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7043	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7044	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7045	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7046	AssertRCReturn(rc, rc);
7047	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7048	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7049	0 /* GCPtrFaultAddress */);
7050	AssertRCReturn(rc, rc);
7051	}
7052	else
7053	{
7054	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7055	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7056	}
7057	break;
7058	}
7059	}
7060	break;
7061	}
7062
7063	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7064	default:
7065	{
7066	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7067	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7068	break;
7069	}
7070	}
7071	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7072	return rc;
7073	}
7074
7075
7076	/**
7077	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7078	*/
7079	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7080	{
7081	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7082
7083	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7084	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7085	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7086	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7087	AssertRCReturn(rc, rc);
7088
7089	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7091	return VINF_SUCCESS;
7092	}
7093
7094
7095	/**
7096	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7097	*/
7098	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7099	{
7100	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7101	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7102	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7103	}
7104
7105
7106	/**
7107	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7108	*/
7109	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7110	{
7111	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7113	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7114	}
7115
7116
7117	/**
7118	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7119	*/
7120	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7121	{
7122	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7124	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7125	}
7126
7127
7128	/**
7129	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7130	*/
7131	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7132	{
7133	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7134	PVM pVM = pVCpu->CTX_SUFF(pVM);
7135	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7136	if (RT_LIKELY(rc == VINF_SUCCESS))
7137	{
7138	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7139	Assert(pVmxTransient->cbInstr == 2);
7140	}
7141	else
7142	{
7143	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7144	rc = VERR_EM_INTERPRETER;
7145	}
7146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7147	return rc;
7148	}
7149
7150
7151	/**
7152	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7153	*/
7154	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7155	{
7156	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7157	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7158	AssertRCReturn(rc, rc);
7159
7160	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7161	return VINF_EM_RAW_EMULATE_INSTR;
7162
7163	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7164	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7165	}
7166
7167
7168	/**
7169	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7170	*/
7171	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7172	{
7173	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7174	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7175	AssertRCReturn(rc, rc);
7176
7177	PVM pVM = pVCpu->CTX_SUFF(pVM);
7178	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7179	if (RT_LIKELY(rc == VINF_SUCCESS))
7180	{
7181	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7182	Assert(pVmxTransient->cbInstr == 2);
7183	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7184	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7185	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7186	}
7187	else
7188	{
7189	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7190	rc = VERR_EM_INTERPRETER;
7191	}
7192	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7193	return rc;
7194	}
7195
7196
7197	/**
7198	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7199	*/
7200	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7201	{
7202	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7203	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7204	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7205	AssertRCReturn(rc, rc);
7206
7207	PVM pVM = pVCpu->CTX_SUFF(pVM);
7208	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7209	if (RT_LIKELY(rc == VINF_SUCCESS))
7210	{
7211	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7212	Assert(pVmxTransient->cbInstr == 3);
7213	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7214	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7215	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7216	}
7217	else
7218	{
7219	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7220	rc = VERR_EM_INTERPRETER;
7221	}
7222	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7223	return rc;
7224	}
7225
7226
7227	/**
7228	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7229	*/
7230	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7231	{
7232	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7233	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7234	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7235	AssertRCReturn(rc, rc);
7236
7237	PVM pVM = pVCpu->CTX_SUFF(pVM);
7238	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7239	if (RT_LIKELY(rc == VINF_SUCCESS))
7240	{
7241	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7242	Assert(pVmxTransient->cbInstr == 2);
7243	}
7244	else
7245	{
7246	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7247	rc = VERR_EM_INTERPRETER;
7248	}
7249	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7250	return rc;
7251	}
7252
7253
7254	/**
7255	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7256	*/
7257	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7258	{
7259	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7260	PVM pVM = pVCpu->CTX_SUFF(pVM);
7261	Assert(!pVM->hm.s.fNestedPaging);
7262
7263	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7264	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7265	AssertRCReturn(rc, rc);
7266
7267	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7268	rc = VBOXSTRICTRC_VAL(rc2);
7269	if (RT_LIKELY(rc == VINF_SUCCESS))
7270	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7271	else
7272	{
7273	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7274	pVmxTransient->uExitQualification, rc));
7275	}
7276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7277	return rc;
7278	}
7279
7280
7281	/**
7282	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7283	*/
7284	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7285	{
7286	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7287	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7288	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7289	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7290	AssertRCReturn(rc, rc);
7291
7292	PVM pVM = pVCpu->CTX_SUFF(pVM);
7293	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7294	if (RT_LIKELY(rc == VINF_SUCCESS))
7295	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7296	else
7297	{
7298	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7299	rc = VERR_EM_INTERPRETER;
7300	}
7301	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7302	return rc;
7303	}
7304
7305
7306	/**
7307	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7308	*/
7309	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7310	{
7311	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7312	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7313	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7314	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7315	AssertRCReturn(rc, rc);
7316
7317	PVM pVM = pVCpu->CTX_SUFF(pVM);
7318	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7319	rc = VBOXSTRICTRC_VAL(rc2);
7320	if (RT_LIKELY( rc == VINF_SUCCESS
7321	\|\| rc == VINF_EM_HALT))
7322	{
7323	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7324	AssertRCReturn(rc3, rc3);
7325
7326	if ( rc == VINF_EM_HALT
7327	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7328	{
7329	rc = VINF_SUCCESS;
7330	}
7331	}
7332	else
7333	{
7334	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7335	rc = VERR_EM_INTERPRETER;
7336	}
7337	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7338	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7339	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7340	return rc;
7341	}
7342
7343
7344	/**
7345	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7346	*/
7347	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7348	{
7349	/*
7350	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7351	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7352	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7353	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7354	*/
7355	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7356	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7357	}
7358
7359
7360	/**
7361	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7362	*/
7363	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7364	{
7365	/*
7366	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7367	* root operation. If we get there there is something funny going on.
7368	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7369	*/
7370	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7371	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7372	}
7373
7374
7375	/**
7376	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7377	*/
7378	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7379	{
7380	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7381	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7382	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7383	}
7384
7385
7386	/**
7387	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7388	*/
7389	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7390	{
7391	/*
7392	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7393	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7394	* See Intel spec. 25.3 "Other Causes of VM-exits".
7395	*/
7396	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7397	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7398	}
7399
7400
7401	/**
7402	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7403	* VM-exit.
7404	*/
7405	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7406	{
7407	/*
7408	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7409	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7410	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7411	*/
7412	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7413	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7414	}
7415
7416
7417	/**
7418	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7419	* VM-exit.
7420	*/
7421	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7422	{
7423	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7424	return VINF_EM_RESET;
7425	}
7426
7427
7428	/**
7429	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7430	*/
7431	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7432	{
7433	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7434	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7435	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7436	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7437	AssertRCReturn(rc, rc);
7438
7439	pMixedCtx->rip++;
7440	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7441	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7442	rc = VINF_SUCCESS;
7443	else
7444	rc = VINF_EM_HALT;
7445
7446	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7447	return rc;
7448	}
7449
7450
7451	/**
7452	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7453	*/
7454	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7455	{
7456	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7457	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7458	return VINF_SUCCESS;
7459	}
7460
7461
7462	/**
7463	* VM-exit handler for expiry of the VMX preemption timer.
7464	*/
7465	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7466	{
7467	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7468
7469	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7470	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7471	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7472
7473	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7474	PVM pVM = pVCpu->CTX_SUFF(pVM);
7475	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7476	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7477	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7478	}
7479
7480
7481	/**
7482	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7483	*/
7484	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7485	{
7486	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7487	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7488	/** @todo check if XSETBV is supported by the recompiler. */
7489	return VERR_EM_INTERPRETER;
7490	}
7491
7492
7493	/**
7494	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7495	*/
7496	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7497	{
7498	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7499	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7500	/** @todo implement EMInterpretInvpcid() */
7501	return VERR_EM_INTERPRETER;
7502	}
7503
7504
7505	/**
7506	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7507	* Error VM-exit.
7508	*/
7509	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7510	{
7511	uint32_t uIntrState;
7512	RTHCUINTREG uHCReg;
7513	uint64_t u64Val;
7514
7515	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7516	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7517	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7518	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7519	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7520	AssertRCReturn(rc, rc);
7521
7522	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7523	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7524	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7525	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7526
7527	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7528	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7529	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7530	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7531	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7532	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7533	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7534	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7535	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7536	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7537	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7538	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7539
7540	PVM pVM = pVCpu->CTX_SUFF(pVM);
7541	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7542
7543	return VERR_VMX_INVALID_GUEST_STATE;
7544	}
7545
7546
7547	/**
7548	* VM-exit handler for VM-entry failure due to an MSR-load
7549	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7550	*/
7551	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7552	{
7553	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7554	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7555	}
7556
7557
7558	/**
7559	* VM-exit handler for VM-entry failure due to a machine-check event
7560	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7561	*/
7562	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7563	{
7564	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7565	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7566	}
7567
7568
7569	/**
7570	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7571	* theory.
7572	*/
7573	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7574	{
7575	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7576	return VERR_VMX_UNDEFINED_EXIT_CODE;
7577	}
7578
7579
7580	/**
7581	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7582	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7583	* Conditional VM-exit.
7584	*/
7585	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7586	{
7587	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7588	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7589	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7590	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7591	return VERR_EM_INTERPRETER;
7592	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7593	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7594	}
7595
7596
7597	/**
7598	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7599	*/
7600	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7601	{
7602	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7603	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7605	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7606	return VERR_EM_INTERPRETER;
7607	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7608	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7609	}
7610
7611
7612	/**
7613	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7614	*/
7615	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7616	{
7617	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7618	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7619	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7620	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7621	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7622	AssertRCReturn(rc, rc);
7623
7624	PVM pVM = pVCpu->CTX_SUFF(pVM);
7625	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7626	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7627	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7629
7630	/* Update RIP and continue guest execution. */
7631	if (RT_LIKELY(rc == VINF_SUCCESS))
7632	{
7633	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7634	Assert(pVmxTransient->cbInstr == 2);
7635	}
7636	return rc;
7637	}
7638
7639
7640	/**
7641	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7642	*/
7643	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7644	{
7645	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7646	PVM pVM = pVCpu->CTX_SUFF(pVM);
7647	int rc = VINF_SUCCESS;
7648
7649	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7650	if ( pVM->hm.s.fTPRPatchingActive
7651	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7652	{
7653	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7654	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7655	{
7656	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7657	AssertRC(rc);
7658	}
7659
7660	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7661	Assert(pVmxTransient->cbInstr == 2);
7662	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7663	return VINF_SUCCESS;
7664	}
7665
7666	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7667	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7668	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7669	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7670	AssertRCReturn(rc, rc);
7671	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7672
7673	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7674	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7675	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7676
7677	/* Update guest-state and continue execution. */
7678	if (RT_LIKELY(rc == VINF_SUCCESS))
7679	{
7680	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7681
7682	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7683	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7684	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7685	{
7686	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7687	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7688	}
7689	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7690	{
7691	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7692	AssertRCReturn(rc, rc);
7693	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7694	}
7695
7696	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7697	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7698	{
7699	switch (pMixedCtx->ecx)
7700	{
7701	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7702	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7703	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7704	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7705	case MSR_K8_FS_BASE: /* no break */
7706	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7707	}
7708	}
7709	#ifdef VBOX_STRICT
7710	else
7711	{
7712	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7713	switch (pMixedCtx->ecx)
7714	{
7715	case MSR_IA32_SYSENTER_CS:
7716	case MSR_IA32_SYSENTER_EIP:
7717	case MSR_IA32_SYSENTER_ESP:
7718	case MSR_K8_FS_BASE:
7719	case MSR_K8_GS_BASE:
7720	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7721	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7722
7723	case MSR_K8_LSTAR:
7724	case MSR_K6_STAR:
7725	case MSR_K8_SF_MASK:
7726	case MSR_K8_TSC_AUX:
7727	{
7728	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7729	pMixedCtx->ecx));
7730	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7731	}
7732
7733	case MSR_K8_KERNEL_GS_BASE:
7734	{
7735	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7736	pMixedCtx->ecx));
7737	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7738	}
7739	}
7740	}
7741	#endif /* VBOX_STRICT */
7742	}
7743	return rc;
7744	}
7745
7746
7747	/**
7748	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7749	*/
7750	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7751	{
7752	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7753	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7755	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7756	return VERR_EM_INTERPRETER;
7757	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7758	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7759	}
7760
7761
7762	/**
7763	* VM-exit handler for when the TPR value is lowered below the specified
7764	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7765	*/
7766	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7767	{
7768	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7769	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7770
7771	/*
7772	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7773	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7774	* resume guest execution.
7775	*/
7776	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7777	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7778	return VINF_SUCCESS;
7779	}
7780
7781
7782	/**
7783	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7784	* VM-exit.
7785	*
7786	* @retval VINF_SUCCESS when guest execution can continue.
7787	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7788	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7789	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7790	* recompiler.
7791	*/
7792	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7793	{
7794	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7795	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7796	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7797	AssertRCReturn(rc, rc);
7798
7799	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7800	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7801	PVM pVM = pVCpu->CTX_SUFF(pVM);
7802	switch (uAccessType)
7803	{
7804	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7805	{
7806	#if 0
7807	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7808	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7809	#else
7810	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7811	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7812	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7813	#endif
7814	AssertRCReturn(rc, rc);
7815
7816	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7817	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7818	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7819	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7820
7821	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7822	{
7823	case 0: /* CR0 */
7824	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7825	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7826	break;
7827	case 2: /* C2 **/
7828	/* Nothing to do here, CR2 it's not part of the VMCS. */
7829	break;
7830	case 3: /* CR3 */
7831	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7832	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7833	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7834	break;
7835	case 4: /* CR4 */
7836	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7837	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7838	break;
7839	case 8: /* CR8 */
7840	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7841	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7842	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7843	break;
7844	default:
7845	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7846	break;
7847	}
7848
7849	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7850	break;
7851	}
7852
7853	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7854	{
7855	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7856	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7857	AssertRCReturn(rc, rc);
7858	Assert( !pVM->hm.s.fNestedPaging
7859	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7860	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7861
7862	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7863	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7864	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7865
7866	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7867	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7868	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7869	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7870	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7871	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7872	break;
7873	}
7874
7875	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7876	{
7877	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7878	AssertRCReturn(rc, rc);
7879	rc = EMInterpretCLTS(pVM, pVCpu);
7880	AssertRCReturn(rc, rc);
7881	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7882	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7883	Log(("CRX CLTS write rc=%d\n", rc));
7884	break;
7885	}
7886
7887	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7888	{
7889	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7890	AssertRCReturn(rc, rc);
7891	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7892	if (RT_LIKELY(rc == VINF_SUCCESS))
7893	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7895	Log(("CRX LMSW write rc=%d\n", rc));
7896	break;
7897	}
7898
7899	default:
7900	{
7901	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7902	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7903	}
7904	}
7905
7906	/* Validate possible error codes. */
7907	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7908	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7909	if (RT_SUCCESS(rc))
7910	{
7911	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7912	AssertRCReturn(rc2, rc2);
7913	}
7914
7915	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7916	return rc;
7917	}
7918
7919
7920	/**
7921	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7922	* VM-exit.
7923	*/
7924	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7925	{
7926	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7927	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7928
7929	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7930	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7931	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7932	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7933	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7934	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7935	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7936	AssertRCReturn(rc, rc);
7937	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7938
7939	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7940	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7941	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7942	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7943	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7944	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7945	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7946
7947	/* I/O operation lookup arrays. */
7948	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7949	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7950
7951	const uint32_t cbSize = s_aIOSize[uIOWidth];
7952	const uint32_t cbInstr = pVmxTransient->cbInstr;
7953	PVM pVM = pVCpu->CTX_SUFF(pVM);
7954	if (fIOString)
7955	{
7956	/* INS/OUTS - I/O String instruction. */
7957	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7958	/** @todo for now manually disassemble later optimize by getting the fields from
7959	* the VMCS. */
7960	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7961	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7962	* segment prefix info. */
7963	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7964	if (RT_SUCCESS(rc))
7965	{
7966	if (fIOWrite)
7967	{
7968	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7969	(DISCPUMODE)pDis->uAddrMode, cbSize);
7970	rc = VBOXSTRICTRC_VAL(rc2);
7971	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7972	}
7973	else
7974	{
7975	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7976	(DISCPUMODE)pDis->uAddrMode, cbSize);
7977	rc = VBOXSTRICTRC_VAL(rc2);
7978	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7979	}
7980	}
7981	else
7982	{
7983	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7984	rc = VINF_EM_RAW_EMULATE_INSTR;
7985	}
7986	}
7987	else
7988	{
7989	/* IN/OUT - I/O instruction. */
7990	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7991	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7992	if (fIOWrite)
7993	{
7994	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7995	rc = VBOXSTRICTRC_VAL(rc2);
7996	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7997	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7998	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7999	}
8000	else
8001	{
8002	uint32_t u32Result = 0;
8003	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8004	rc = VBOXSTRICTRC_VAL(rc2);
8005	if (IOM_SUCCESS(rc))
8006	{
8007	/* Save result of I/O IN instr. in AL/AX/EAX. */
8008	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8009	}
8010	else if (rc == VINF_IOM_R3_IOPORT_READ)
8011	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8012	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8013	}
8014	}
8015
8016	if (IOM_SUCCESS(rc))
8017	{
8018	pMixedCtx->rip += cbInstr;
8019	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8020	if (RT_LIKELY(rc == VINF_SUCCESS))
8021	{
8022	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8023	AssertRCReturn(rc, rc);
8024
8025	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8026	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8027	{
8028	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8029	for (unsigned i = 0; i < 4; i++)
8030	{
8031	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8032	if ( ( uIOPort >= pMixedCtx->dr[i]
8033	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8034	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8035	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8036	{
8037	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8038	uint64_t uDR6 = ASMGetDR6();
8039
8040	/* Clear all breakpoint status flags and set the one we just hit. */
8041	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8042	uDR6 \|= (uint64_t)RT_BIT(i);
8043
8044	/*
8045	* Note: AMD64 Architecture Programmer's Manual 13.1:
8046	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8047	* be cleared by software after the contents have been read.
8048	*/
8049	ASMSetDR6(uDR6);
8050
8051	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8052	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8053
8054	/* Paranoia. */
8055	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8056	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8057	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8058
8059	/* Resync DR7 */
8060	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8061	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8062
8063	/* Set #DB to be injected into the VM and continue guest execution. */
8064	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8065	break;
8066	}
8067	}
8068	}
8069	}
8070	}
8071
8072	#ifdef DEBUG
8073	if (rc == VINF_IOM_R3_IOPORT_READ)
8074	Assert(!fIOWrite);
8075	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8076	Assert(fIOWrite);
8077	else
8078	{
8079	AssertMsg( RT_FAILURE(rc)
8080	\|\| rc == VINF_SUCCESS
8081	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8082	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8083	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8084	}
8085	#endif
8086
8087	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8088	return rc;
8089	}
8090
8091
8092	/**
8093	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8094	* VM-exit.
8095	*/
8096	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8097	{
8098	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8099
8100	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8101	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8102	AssertRCReturn(rc, rc);
8103	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8104	{
8105	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8106	AssertRCReturn(rc, rc);
8107	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8108	{
8109	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8110	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8111	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8112	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8113	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8114	{
8115	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8116	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8117
8118	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8119	Assert(!pVCpu->hm.s.Event.fPending);
8120	pVCpu->hm.s.Event.fPending = true;
8121	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8122	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8123	AssertRCReturn(rc, rc);
8124	if (fErrorCodeValid)
8125	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8126	else
8127	pVCpu->hm.s.Event.u32ErrCode = 0;
8128	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8129	&& uVector == X86_XCPT_PF)
8130	{
8131	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8132	}
8133	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8134	}
8135	}
8136	}
8137	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8138	* emulation. */
8139	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8140	return VERR_EM_INTERPRETER;
8141	}
8142
8143
8144	/**
8145	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8146	*/
8147	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8148	{
8149	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8150	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8151	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8152	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8153	AssertRCReturn(rc, rc);
8154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8155	return VINF_EM_DBG_STOP;
8156	}
8157
8158
8159	/**
8160	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8161	*/
8162	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8163	{
8164	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8165
8166	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8167	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8168	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8169	return VINF_SUCCESS;
8170	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8171	return rc;
8172
8173	#if 0
8174	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8175	* just sync the whole thing. */
8176	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8177	#else
8178	/* Aggressive state sync. for now. */
8179	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8180	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8181	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8182	#endif
8183	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8184	AssertRCReturn(rc, rc);
8185
8186	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8187	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8188	switch (uAccessType)
8189	{
8190	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8191	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8192	{
8193	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8194	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8195	{
8196	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8197	}
8198
8199	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8200	GCPhys &= PAGE_BASE_GC_MASK;
8201	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8202	PVM pVM = pVCpu->CTX_SUFF(pVM);
8203	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8204	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8205
8206	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8207	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8208	CPUMCTX2CORE(pMixedCtx), GCPhys);
8209	rc = VBOXSTRICTRC_VAL(rc2);
8210	Log(("ApicAccess rc=%d\n", rc));
8211	if ( rc == VINF_SUCCESS
8212	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8213	\|\| rc == VERR_PAGE_NOT_PRESENT)
8214	{
8215	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8216	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8217	rc = VINF_SUCCESS;
8218	}
8219	break;
8220	}
8221
8222	default:
8223	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8224	rc = VINF_EM_RAW_EMULATE_INSTR;
8225	break;
8226	}
8227
8228	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8229	return rc;
8230	}
8231
8232
8233	/**
8234	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8235	* VM-exit.
8236	*/
8237	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8238	{
8239	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8240
8241	/* We should -not- get this VM-exit if the guest is debugging. */
8242	if (CPUMIsGuestDebugStateActive(pVCpu))
8243	{
8244	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8245	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8246	}
8247
8248	int rc = VERR_INTERNAL_ERROR_5;
8249	if ( !DBGFIsStepping(pVCpu)
8250	&& !CPUMIsHyperDebugStateActive(pVCpu))
8251	{
8252	/* Don't intercept MOV DRx. */
8253	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8254	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8255	AssertRCReturn(rc, rc);
8256
8257	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8258	PVM pVM = pVCpu->CTX_SUFF(pVM);
8259	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8260	AssertRC(rc);
8261	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8262
8263	#ifdef VBOX_WITH_STATISTICS
8264	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8265	AssertRCReturn(rc, rc);
8266	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8268	else
8269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8270	#endif
8271	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8272	return VINF_SUCCESS;
8273	}
8274
8275	/*
8276	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8277	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8278	*/
8279	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8280	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8281	AssertRCReturn(rc, rc);
8282
8283	PVM pVM = pVCpu->CTX_SUFF(pVM);
8284	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8285	{
8286	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8287	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8288	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8289	if (RT_SUCCESS(rc))
8290	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8292	}
8293	else
8294	{
8295	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8296	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8297	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8299	}
8300
8301	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8302	if (RT_SUCCESS(rc))
8303	{
8304	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8305	AssertRCReturn(rc2, rc2);
8306	}
8307	return rc;
8308	}
8309
8310
8311	/**
8312	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8313	* Conditional VM-exit.
8314	*/
8315	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8316	{
8317	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8318	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8319
8320	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8321	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8322	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8323	return VINF_SUCCESS;
8324	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8325	return rc;
8326
8327	RTGCPHYS GCPhys = 0;
8328	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8329
8330	#if 0
8331	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8332	#else
8333	/* Aggressive state sync. for now. */
8334	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8335	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8336	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8337	#endif
8338	AssertRCReturn(rc, rc);
8339
8340	/*
8341	* If we succeed, resume guest execution.
8342	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8343	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8344	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8345	* weird case. See @bugref{6043}.
8346	*/
8347	PVM pVM = pVCpu->CTX_SUFF(pVM);
8348	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8349	rc = VBOXSTRICTRC_VAL(rc2);
8350	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8351	if ( rc == VINF_SUCCESS
8352	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8353	\|\| rc == VERR_PAGE_NOT_PRESENT)
8354	{
8355	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8356	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8357	return VINF_SUCCESS;
8358	}
8359	return rc;
8360	}
8361
8362
8363	/**
8364	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8365	* VM-exit.
8366	*/
8367	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8368	{
8369	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8370	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8371
8372	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8373	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8374	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8375	return VINF_SUCCESS;
8376	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8377	return rc;
8378
8379	RTGCPHYS GCPhys = 0;
8380	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8381	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8382	#if 0
8383	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8384	#else
8385	/* Aggressive state sync. for now. */
8386	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8387	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8388	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8389	#endif
8390	AssertRCReturn(rc, rc);
8391
8392	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8393	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8394
8395	RTGCUINT uErrorCode = 0;
8396	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8397	uErrorCode \|= X86_TRAP_PF_ID;
8398	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8399	uErrorCode \|= X86_TRAP_PF_RW;
8400	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8401	uErrorCode \|= X86_TRAP_PF_P;
8402
8403	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8404
8405	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8406	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8407
8408	/* Handle the pagefault trap for the nested shadow table. */
8409	PVM pVM = pVCpu->CTX_SUFF(pVM);
8410	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8411	TRPMResetTrap(pVCpu);
8412
8413	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8414	if ( rc == VINF_SUCCESS
8415	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8416	\|\| rc == VERR_PAGE_NOT_PRESENT)
8417	{
8418	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8419	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8420	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8421	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8422	return VINF_SUCCESS;
8423	}
8424
8425	Log(("EPT return to ring-3 rc=%d\n"));
8426	return rc;
8427	}
8428
8429
8430	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8431	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8432	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8433	/**
8434	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8435	*/
8436	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8437	{
8438	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8439	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8440
8441	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8442	AssertRCReturn(rc, rc);
8443
8444	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8445	{
8446	/* Old-style FPU error reporting needs some extra work. */
8447	/** @todo don't fall back to the recompiler, but do it manually. */
8448	return VERR_EM_INTERPRETER;
8449	}
8450	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8451	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8452	return rc;
8453	}
8454
8455
8456	/**
8457	* VM-exit exception handler for #BP (Breakpoint exception).
8458	*/
8459	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8460	{
8461	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8463
8464	/** @todo Try optimize this by not saving the entire guest state unless
8465	* really needed. */
8466	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8467	AssertRCReturn(rc, rc);
8468
8469	PVM pVM = pVCpu->CTX_SUFF(pVM);
8470	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8471	if (rc == VINF_EM_RAW_GUEST_TRAP)
8472	{
8473	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8474	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8475	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8476	AssertRCReturn(rc, rc);
8477
8478	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8479	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8480	}
8481
8482	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8483	return rc;
8484	}
8485
8486
8487	/**
8488	* VM-exit exception handler for #DB (Debug exception).
8489	*/
8490	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8491	{
8492	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8493	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8494
8495	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8496	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8497	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8498	AssertRCReturn(rc, rc);
8499
8500	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8501	uint64_t uDR6 = X86_DR6_INIT_VAL;
8502	uDR6 \|= (pVmxTransient->uExitQualification
8503	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8504	PVM pVM = pVCpu->CTX_SUFF(pVM);
8505	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8506	if (rc == VINF_EM_RAW_GUEST_TRAP)
8507	{
8508	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8509	pMixedCtx->dr[6] = uDR6;
8510
8511	if (CPUMIsGuestDebugStateActive(pVCpu))
8512	ASMSetDR6(pMixedCtx->dr[6]);
8513
8514	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8515
8516	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8517	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8518
8519	/* Paranoia. */
8520	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8521	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8522	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8523
8524	/* Resync DR7. */
8525	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8526	AssertRCReturn(rc,rc);
8527	}
8528
8529	/*
8530	* If the #DB exception was meant for the guest, reflect it to the guest upon VM-reentry. If our hypervisor is
8531	* simultaneously single-stepping with the guest, return to the debugger but also reflect #DB to the guest upon VM-reentry.
8532	*/
8533	if ( rc == VINF_EM_RAW_GUEST_TRAP
8534	\|\| rc == VINF_EM_DBG_STEPPED)
8535	{
8536	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8537	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8538	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8539	AssertRCReturn(rc2, rc2);
8540	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8541	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8542	if (rc == VINF_EM_RAW_GUEST_TRAP)
8543	rc = VINF_SUCCESS;
8544	}
8545	return rc;
8546	}
8547
8548
8549	/**
8550	* VM-exit exception handler for #NM (Device-not-available exception: floating
8551	* point exception).
8552	*/
8553	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8554	{
8555	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8556
8557	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8558	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8559	#endif
8560
8561	/* We require CR0 and EFER. EFER is always up-to-date. */
8562	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8563	AssertRCReturn(rc, rc);
8564
8565	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8566	PVM pVM = pVCpu->CTX_SUFF(pVM);
8567	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8568	if (rc == VINF_SUCCESS)
8569	{
8570	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8571	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8572	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8573	return VINF_SUCCESS;
8574	}
8575
8576	/* Forward #NM to the guest. */
8577	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8578	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8579	AssertRCReturn(rc, rc);
8580	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8581	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8583	return rc;
8584	}
8585
8586
8587	/**
8588	* VM-exit exception handler for #GP (General-protection exception).
8589	*
8590	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8591	*/
8592	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8593	{
8594	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8596
8597	int rc = VERR_INTERNAL_ERROR_5;
8598
8599	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8600	{
8601	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8602	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8603	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8604	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8605	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8606	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8607	AssertRCReturn(rc, rc);
8608	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8609	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8610	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8611	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8612	return rc;
8613	#else
8614	/* We don't intercept #GP. */
8615	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8616	return VERR_VMX_UNEXPECTED_EXCEPTION;
8617	#endif
8618	}
8619
8620	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8621	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8622
8623	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8624	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8625	AssertRCReturn(rc, rc);
8626
8627	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8628	uint32_t cbOp = 0;
8629	PVM pVM = pVCpu->CTX_SUFF(pVM);
8630	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8631	if (RT_SUCCESS(rc))
8632	{
8633	rc = VINF_SUCCESS;
8634	Assert(cbOp == pDis->cbInstr);
8635	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8636	switch (pDis->pCurInstr->uOpcode)
8637	{
8638	case OP_CLI:
8639	pMixedCtx->eflags.Bits.u1IF = 0;
8640	pMixedCtx->rip += pDis->cbInstr;
8641	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8643	break;
8644
8645	case OP_STI:
8646	pMixedCtx->eflags.Bits.u1IF = 1;
8647	pMixedCtx->rip += pDis->cbInstr;
8648	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8649	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8650	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8652	break;
8653
8654	case OP_HLT:
8655	rc = VINF_EM_HALT;
8656	pMixedCtx->rip += pDis->cbInstr;
8657	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8658	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8659	break;
8660
8661	case OP_POPF:
8662	{
8663	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8664	uint32_t cbParm = 0;
8665	uint32_t uMask = 0;
8666	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8667	{
8668	cbParm = 4;
8669	uMask = 0xffffffff;
8670	}
8671	else
8672	{
8673	cbParm = 2;
8674	uMask = 0xffff;
8675	}
8676
8677	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8678	RTGCPTR GCPtrStack = 0;
8679	X86EFLAGS uEflags;
8680	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8681	&GCPtrStack);
8682	if (RT_SUCCESS(rc))
8683	{
8684	Assert(sizeof(uEflags.u32) >= cbParm);
8685	uEflags.u32 = 0;
8686	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8687	}
8688	if (RT_FAILURE(rc))
8689	{
8690	rc = VERR_EM_INTERPRETER;
8691	break;
8692	}
8693	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8694	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8695	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8696	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8697	pMixedCtx->eflags.Bits.u1RF = 0;
8698	pMixedCtx->esp += cbParm;
8699	pMixedCtx->esp &= uMask;
8700	pMixedCtx->rip += pDis->cbInstr;
8701	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8703	break;
8704	}
8705
8706	case OP_PUSHF:
8707	{
8708	uint32_t cbParm = 0;
8709	uint32_t uMask = 0;
8710	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8711	{
8712	cbParm = 4;
8713	uMask = 0xffffffff;
8714	}
8715	else
8716	{
8717	cbParm = 2;
8718	uMask = 0xffff;
8719	}
8720
8721	/* Get the stack pointer & push the contents of eflags onto the stack. */
8722	RTGCPTR GCPtrStack = 0;
8723	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8724	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8725	if (RT_FAILURE(rc))
8726	{
8727	rc = VERR_EM_INTERPRETER;
8728	break;
8729	}
8730	X86EFLAGS uEflags;
8731	uEflags = pMixedCtx->eflags;
8732	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8733	uEflags.Bits.u1RF = 0;
8734	uEflags.Bits.u1VM = 0;
8735
8736	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8737	if (RT_FAILURE(rc))
8738	{
8739	rc = VERR_EM_INTERPRETER;
8740	break;
8741	}
8742	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8743	pMixedCtx->esp -= cbParm;
8744	pMixedCtx->esp &= uMask;
8745	pMixedCtx->rip += pDis->cbInstr;
8746	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8747	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8748	break;
8749	}
8750
8751	case OP_IRET:
8752	{
8753	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8754	* instruction reference. */
8755	RTGCPTR GCPtrStack = 0;
8756	uint32_t uMask = 0xffff;
8757	uint16_t aIretFrame[3];
8758	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8759	{
8760	rc = VERR_EM_INTERPRETER;
8761	break;
8762	}
8763	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8764	&GCPtrStack);
8765	if (RT_SUCCESS(rc))
8766	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8767	if (RT_FAILURE(rc))
8768	{
8769	rc = VERR_EM_INTERPRETER;
8770	break;
8771	}
8772	pMixedCtx->eip = 0;
8773	pMixedCtx->ip = aIretFrame[0];
8774	pMixedCtx->cs.Sel = aIretFrame[1];
8775	pMixedCtx->cs.ValidSel = aIretFrame[1];
8776	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8777	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8778	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8779	pMixedCtx->sp += sizeof(aIretFrame);
8780	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8781	\| HM_CHANGED_GUEST_RFLAGS;
8782	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8783	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8784	break;
8785	}
8786
8787	case OP_INT:
8788	{
8789	uint16_t uVector = pDis->Param1.uValue & 0xff;
8790	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8791	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8792	break;
8793	}
8794
8795	case OP_INTO:
8796	{
8797	if (pMixedCtx->eflags.Bits.u1OF)
8798	{
8799	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8800	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8801	}
8802	break;
8803	}
8804
8805	default:
8806	{
8807	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8808	EMCODETYPE_SUPERVISOR);
8809	rc = VBOXSTRICTRC_VAL(rc2);
8810	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8811	Log(("#GP rc=%Rrc\n", rc));
8812	break;
8813	}
8814	}
8815	}
8816	else
8817	rc = VERR_EM_INTERPRETER;
8818
8819	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8820	("#GP Unexpected rc=%Rrc\n", rc));
8821	return rc;
8822	}
8823
8824
8825	/**
8826	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8827	* the exception reported in the VMX transient structure back into the VM.
8828	*
8829	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8830	* VMX transient structure to be up-to-date.
8831	*/
8832	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8833	{
8834	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8835
8836	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8837	hmR0VmxCheckExitDueToEventDelivery(). */
8838	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8839	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8840	AssertRCReturn(rc, rc);
8841	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8842	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8843	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8844	return VINF_SUCCESS;
8845	}
8846
8847
8848	/**
8849	* VM-exit exception handler for #PF (Page-fault exception).
8850	*/
8851	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8852	{
8853	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8854	PVM pVM = pVCpu->CTX_SUFF(pVM);
8855	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8856	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8857	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8858	AssertRCReturn(rc, rc);
8859
8860	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8861	if (pVM->hm.s.fNestedPaging)
8862	{
8863	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8864	{
8865	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8866	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8867	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8868	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8869	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8870	}
8871	else
8872	{
8873	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8874	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8875	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8876	Log(("Pending #DF due to vectoring #PF. NP\n"));
8877	}
8878	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8879	return rc;
8880	}
8881	#else
8882	Assert(!pVM->hm.s.fNestedPaging);
8883	#endif
8884
8885	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8886	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8887	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8888	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8889	AssertRCReturn(rc, rc);
8890	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8891	if ( pVM->hm.s.fTRPPatchingAllowed
8892	&& pVM->hm.s.pGuestPatchMem
8893	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8894	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8895	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8896	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8897	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8898	{
8899	RTGCPHYS GCPhys;
8900	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8901	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8902	if ( rc == VINF_SUCCESS
8903	&& GCPhys == GCPhysApicBase)
8904	{
8905	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8906	AssertRCReturn(rc, rc);
8907
8908	/* Only attempt to patch the instruction once. */
8909	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8910	if (!pPatch)
8911	return VINF_EM_HM_PATCH_TPR_INSTR;
8912	}
8913	}
8914	#endif
8915
8916	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8917	AssertRCReturn(rc, rc);
8918
8919	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8920	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8921
8922	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8923	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8924	(RTGCPTR)pVmxTransient->uExitQualification);
8925
8926	Log(("#PF: rc=%Rrc\n", rc));
8927	if (rc == VINF_SUCCESS)
8928	{
8929	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8930	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8931	* memory? We don't update the whole state here... */
8932	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8933	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8934	TRPMResetTrap(pVCpu);
8935	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8936	return rc;
8937	}
8938	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8939	{
8940	if (!pVmxTransient->fVectoringPF)
8941	{
8942	/* It's a guest page fault and needs to be reflected to the guest. */
8943	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8944	TRPMResetTrap(pVCpu);
8945	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8946	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8947	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8948	0 /* cbInstr */, uGstErrorCode, pMixedCtx->cr2);
8949	}
8950	else
8951	{
8952	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8953	TRPMResetTrap(pVCpu);
8954	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8955	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8956	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8957	}
8958
8959	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8960	return VINF_SUCCESS;
8961	}
8962
8963	TRPMResetTrap(pVCpu);
8964	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8965	return rc;
8966	}
8967

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45639

Download in other formats: