HMVMXR0.cpp@ 45857

Last change on this file since 45857 was 45857, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Eliminated some unnecessary DECLCALLBACK calling convention.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.3 KB

Line
1	/* $Id: HMVMXR0.cpp 45857 2013-05-01 00:17:17Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147
148	/*******************************************************************************
149	* Structures and Typedefs *
150	*******************************************************************************/
151	/**
152	* A state structure for holding miscellaneous information across
153	* VMX non-root operation and restored after the transition.
154	*/
155	typedef struct VMXTRANSIENT
156	{
157	/** The host's rflags/eflags. */
158	RTCCUINTREG uEFlags;
159	#if HC_ARCH_BITS == 32
160	uint32_t u32Alignment0;
161	#endif
162	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
163	uint64_t u64LStarMsr;
164	/** The guest's TPR value used for TPR shadowing. */
165	uint8_t u8GuestTpr;
166	/** Alignment. */
167	uint8_t abAlignment0[6];
168
169	/** The basic VM-exit reason. */
170	uint16_t uExitReason;
171	/** Alignment. */
172	uint16_t u16Alignment0;
173	/** The VM-exit interruption error code. */
174	uint32_t uExitIntrErrorCode;
175	/** The VM-exit exit qualification. */
176	RTGCUINTPTR uExitQualification;
177	#if GC_ARCH_BITS == 32
178	/** Alignment. */
179	uint32_t u32Alignment1;
180	#endif
181
182	/** The VM-exit interruption-information field. */
183	uint32_t uExitIntrInfo;
184	/** The VM-exit instruction-length field. */
185	uint32_t cbInstr;
186	/** Whether the VM-entry failed or not. */
187	bool fVMEntryFailed;
188	/** Alignment. */
189	uint8_t abAlignment1[5];
190
191	/** The VM-entry interruption-information field. */
192	uint32_t uEntryIntrInfo;
193	/** The VM-entry exception error code field. */
194	uint32_t uEntryXcptErrorCode;
195	/** The VM-entry instruction length field. */
196	uint32_t cbEntryInstr;
197
198	/** IDT-vectoring information field. */
199	uint32_t uIdtVectoringInfo;
200	/** IDT-vectoring error code. */
201	uint32_t uIdtVectoringErrorCode;
202
203	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
204	uint32_t fVmcsFieldsRead;
205	/** Whether TSC-offsetting should be setup before VM-entry. */
206	bool fUpdateTscOffsettingAndPreemptTimer;
207	/** Whether the VM-exit was caused by a page-fault during delivery of a
208	* contributary exception or a page-fault. */
209	bool fVectoringPF;
210	} VMXTRANSIENT, *PVMXTRANSIENT;
211	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
212	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
213	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
214
215
216	/**
217	* MSR-bitmap read permissions.
218	*/
219	typedef enum VMXMSREXITREAD
220	{
221	/** Reading this MSR causes a VM-exit. */
222	VMXMSREXIT_INTERCEPT_READ = 0xb,
223	/** Reading this MSR does not cause a VM-exit. */
224	VMXMSREXIT_PASSTHRU_READ
225	} VMXMSREXITREAD;
226
227	/**
228	* MSR-bitmap write permissions.
229	*/
230	typedef enum VMXMSREXITWRITE
231	{
232	/** Writing to this MSR causes a VM-exit. */
233	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
234	/** Writing to this MSR does not cause a VM-exit. */
235	VMXMSREXIT_PASSTHRU_WRITE
236	} VMXMSREXITWRITE;
237
238
239	/*******************************************************************************
240	* Internal Functions *
241	*******************************************************************************/
242	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
243	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
244	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
245	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
246	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
247	#endif
248	#ifndef HMVMX_USE_FUNCTION_TABLE
249	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
250	#endif
251
252	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296
297	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304
305
306	/*******************************************************************************
307	* Global Variables *
308	*******************************************************************************/
309	#ifdef HMVMX_USE_FUNCTION_TABLE
310	/**
311	* VM-exit handler.
312	*
313	* @returns VBox status code.
314	* @param pVCpu Pointer to the VMCPU.
315	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
316	* out-of-sync. Make sure to update the required
317	* fields before using them.
318	* @param pVmxTransient Pointer to the VMX-transient structure.
319	*/
320	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	/** Pointer to VM-exit handler. */
322	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
323
324	/**
325	* VMX_EXIT dispatch table.
326	*/
327	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
328	{
329	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
330	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
331	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
332	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
333	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
334	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
335	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
336	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
337	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
338	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
339	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
340	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
341	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
342	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
343	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
344	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
345	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
346	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
347	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
348	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
349	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
350	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
351	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
352	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
353	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
354	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
355	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
356	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
357	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
358	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
359	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
360	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
361	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
362	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
363	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
364	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
365	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
366	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
367	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
368	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
369	/* 40 UNDEFINED */ hmR0VmxExitPause,
370	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
371	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
372	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
373	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
374	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
376	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
377	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
378	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
379	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
380	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
381	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
382	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
383	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
384	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
385	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
387	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
388	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
389	};
390	#endif /* HMVMX_USE_FUNCTION_TABLE */
391
392	#ifdef VBOX_STRICT
393	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
394	{
395	/* 0 */ "(Not Used)",
396	/* 1 */ "VMCALL executed in VMX root operation.",
397	/* 2 */ "VMCLEAR with invalid physical address.",
398	/* 3 */ "VMCLEAR with VMXON pointer.",
399	/* 4 */ "VMLAUNCH with non-clear VMCS.",
400	/* 5 */ "VMRESUME with non-launched VMCS.",
401	/* 6 */ "VMRESUME after VMXOFF",
402	/* 7 */ "VM entry with invalid control fields.",
403	/* 8 */ "VM entry with invalid host state fields.",
404	/* 9 */ "VMPTRLD with invalid physical address.",
405	/* 10 */ "VMPTRLD with VMXON pointer.",
406	/* 11 */ "VMPTRLD with incorrect revision identifier.",
407	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
408	/* 13 */ "VMWRITE to read-only VMCS component.",
409	/* 14 */ "(Not Used)",
410	/* 15 */ "VMXON executed in VMX root operation.",
411	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
412	/* 17 */ "VM entry with non-launched executing VMCS.",
413	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
414	/* 19 */ "VMCALL with non-clear VMCS.",
415	/* 20 */ "VMCALL with invalid VM-exit control fields.",
416	/* 21 */ "(Not Used)",
417	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
418	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
419	/* 24 */ "VMCALL with invalid SMM-monitor features.",
420	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
421	/* 26 */ "VM entry with events blocked by MOV SS.",
422	/* 27 */ "(Not Used)",
423	/* 28 */ "Invalid operand to INVEPT/INVVPID."
424	};
425	#endif /* VBOX_STRICT */
426
427
428
429	/**
430	* Updates the VM's last error record. If there was a VMX instruction error,
431	* reads the error data from the VMCS and updates VCPU's last error record as
432	* well.
433	*
434	* @param pVM Pointer to the VM.
435	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
436	* VERR_VMX_UNABLE_TO_START_VM or
437	* VERR_VMX_INVALID_VMCS_FIELD).
438	* @param rc The error code.
439	*/
440	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
441	{
442	AssertPtr(pVM);
443	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
444	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
445	{
446	AssertPtrReturnVoid(pVCpu);
447	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
448	}
449	pVM->hm.s.lLastError = rc;
450	}
451
452
453	/**
454	* Reads the VM-entry interruption-information field from the VMCS into the VMX
455	* transient structure.
456	*
457	* @returns VBox status code.
458	* @param pVmxTransient Pointer to the VMX transient structure.
459	*
460	* @remarks No-long-jump zone!!!
461	*/
462	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
463	{
464	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
465	AssertRCReturn(rc, rc);
466	return VINF_SUCCESS;
467	}
468
469
470	/**
471	* Reads the VM-entry exception error code field from the VMCS into
472	* the VMX transient structure.
473	*
474	* @returns VBox status code.
475	* @param pVmxTransient Pointer to the VMX transient structure.
476	*
477	* @remarks No-long-jump zone!!!
478	*/
479	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
480	{
481	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
482	AssertRCReturn(rc, rc);
483	return VINF_SUCCESS;
484	}
485
486
487	/**
488	* Reads the VM-entry exception error code field from the VMCS into
489	* the VMX transient structure.
490	*
491	* @returns VBox status code.
492	* @param pVCpu Pointer to the VMCPU.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*
495	* @remarks No-long-jump zone!!!
496	*/
497	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
500	AssertRCReturn(rc, rc);
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption-information field from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit interruption error code from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the VM-exit instruction length field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVCpu Pointer to the VMCPU.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*/
553	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
554	{
555	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
558	AssertRCReturn(rc, rc);
559	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
560	}
561	return VINF_SUCCESS;
562	}
563
564
565	/**
566	* Reads the exit qualification from the VMCS into the VMX transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVCpu Pointer to the VMCPU.
570	* @param pVmxTransient Pointer to the VMX transient structure.
571	*/
572	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
573	{
574	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
575	{
576	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
577	AssertRCReturn(rc, rc);
578	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
579	}
580	return VINF_SUCCESS;
581	}
582
583
584	/**
585	* Reads the IDT-vectoring information field from the VMCS into the VMX
586	* transient structure.
587	*
588	* @returns VBox status code.
589	* @param pVmxTransient Pointer to the VMX transient structure.
590	*
591	* @remarks No-long-jump zone!!!
592	*/
593	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
594	{
595	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
596	{
597	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
598	AssertRCReturn(rc, rc);
599	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
600	}
601	return VINF_SUCCESS;
602	}
603
604
605	/**
606	* Reads the IDT-vectoring error code from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*/
612	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
613	{
614	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
617	AssertRCReturn(rc, rc);
618	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
619	}
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Enters VMX root mode operation on the current CPU.
626	*
627	* @returns VBox status code.
628	* @param pVM Pointer to the VM (optional, can be NULL, after
629	* a resume).
630	* @param HCPhysCpuPage Physical address of the VMXON region.
631	* @param pvCpuPage Pointer to the VMXON region.
632	*/
633	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
634	{
635	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
636	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
637	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
638
639	if (pVM)
640	{
641	/* Write the VMCS revision dword to the VMXON region. */
642	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
643	}
644
645	/* Enable the VMX bit in CR4 if necessary. */
646	RTCCUINTREG uCr4 = ASMGetCR4();
647	if (!(uCr4 & X86_CR4_VMXE))
648	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
649
650	/* Enter VMX root mode. */
651	int rc = VMXEnable(HCPhysCpuPage);
652	if (RT_FAILURE(rc))
653	ASMSetCR4(uCr4);
654
655	return rc;
656	}
657
658
659	/**
660	* Exits VMX root mode operation on the current CPU.
661	*
662	* @returns VBox status code.
663	*/
664	static int hmR0VmxLeaveRootMode(void)
665	{
666	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
667
668	/* If we're for some reason not in VMX root mode, then don't leave it. */
669	if (ASMGetCR4() & X86_CR4_VMXE)
670	{
671	/* Exit VMX root mode and clear the VMX bit in CR4 */
672	VMXDisable();
673	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
674	return VINF_SUCCESS;
675	}
676
677	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
678	}
679
680
681	/**
682	* Allocates and maps one physically contiguous page. The allocated page is
683	* zero'd out. (Used by various VT-x structures).
684	*
685	* @returns IPRT status code.
686	* @param pMemObj Pointer to the ring-0 memory object.
687	* @param ppVirt Where to store the virtual address of the
688	* allocation.
689	* @param pPhys Where to store the physical address of the
690	* allocation.
691	*/
692	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
693	{
694	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
695	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
696	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
697
698	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
699	if (RT_FAILURE(rc))
700	return rc;
701	ppVirt = RTR0MemObjAddress(pMemObj);
702	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
703	ASMMemZero32(*ppVirt, PAGE_SIZE);
704	return VINF_SUCCESS;
705	}
706
707
708	/**
709	* Frees and unmaps an allocated physical page.
710	*
711	* @param pMemObj Pointer to the ring-0 memory object.
712	* @param ppVirt Where to re-initialize the virtual address of
713	* allocation as 0.
714	* @param pHCPhys Where to re-initialize the physical address of the
715	* allocation as 0.
716	*/
717	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
718	{
719	AssertPtr(pMemObj);
720	AssertPtr(ppVirt);
721	AssertPtr(pHCPhys);
722	if (*pMemObj != NIL_RTR0MEMOBJ)
723	{
724	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
725	AssertRC(rc);
726	*pMemObj = NIL_RTR0MEMOBJ;
727	*ppVirt = 0;
728	*pHCPhys = 0;
729	}
730	}
731
732
733	/**
734	* Worker function to free VT-x related structures.
735	*
736	* @returns IPRT status code.
737	* @param pVM Pointer to the VM.
738	*/
739	static void hmR0VmxStructsFree(PVM pVM)
740	{
741	for (VMCPUID i = 0; i < pVM->cCpus; i++)
742	{
743	PVMCPU pVCpu = &pVM->aCpus[i];
744	AssertPtr(pVCpu);
745
746	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
747	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
748	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
749	#endif
750
751	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
752	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
753
754	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
755	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
756	}
757
758	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
759	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
760	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
761	#endif
762	}
763
764
765	/**
766	* Worker function to allocate VT-x related VM structures.
767	*
768	* @returns IPRT status code.
769	* @param pVM Pointer to the VM.
770	*/
771	static int hmR0VmxStructsAlloc(PVM pVM)
772	{
773	/*
774	* Initialize members up-front so we can cleanup properly on allocation failure.
775	*/
776	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
777	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
778	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
779	pVM->hm.s.vmx.HCPhys##a_Name = 0;
780
781	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
782	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
783	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
784	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
785
786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
787	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
788	#endif
789	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
790
791	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
792	for (VMCPUID i = 0; i < pVM->cCpus; i++)
793	{
794	PVMCPU pVCpu = &pVM->aCpus[i];
795	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
796	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
797	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
798	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
799	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
800	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
801	#endif
802	}
803	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
804	#undef VMXLOCAL_INIT_VM_MEMOBJ
805
806	/*
807	* Allocate all the VT-x structures.
808	*/
809	int rc = VINF_SUCCESS;
810	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
811	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
812	if (RT_FAILURE(rc))
813	goto cleanup;
814	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
815	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
816	#endif
817
818	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
820	{
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
822	&pVM->hm.s.vmx.HCPhysApicAccess);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	}
826
827	/*
828	* Initialize per-VCPU VT-x structures.
829	*/
830	for (VMCPUID i =0; i < pVM->cCpus; i++)
831	{
832	PVMCPU pVCpu = &pVM->aCpus[i];
833	AssertPtr(pVCpu);
834
835	/* Allocate the VM control structure (VMCS). */
836	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
837	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840
841	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
842	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
843	{
844	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
845	&pVCpu->hm.s.vmx.HCPhysVirtApic);
846	if (RT_FAILURE(rc))
847	goto cleanup;
848	}
849
850	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
852	{
853	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
854	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
855	if (RT_FAILURE(rc))
856	goto cleanup;
857	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
858	}
859
860	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
861	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
862	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
863	if (RT_FAILURE(rc))
864	goto cleanup;
865
866	/* Allocate the VM-exit MSR-load page for the host MSRs. */
867	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
868	if (RT_FAILURE(rc))
869	goto cleanup;
870	#endif
871	}
872
873	return VINF_SUCCESS;
874
875	cleanup:
876	hmR0VmxStructsFree(pVM);
877	return rc;
878	}
879
880
881	/**
882	* Does global VT-x initialization (called during module initialization).
883	*
884	* @returns VBox status code.
885	*/
886	VMMR0DECL(int) VMXR0GlobalInit(void)
887	{
888	#ifdef HMVMX_USE_FUNCTION_TABLE
889	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
890	# ifdef VBOX_STRICT
891	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
892	Assert(g_apfnVMExitHandlers[i]);
893	# endif
894	#endif
895	return VINF_SUCCESS;
896	}
897
898
899	/**
900	* Does global VT-x termination (called during module termination).
901	*/
902	VMMR0DECL(void) VMXR0GlobalTerm()
903	{
904	/* Nothing to do currently. */
905	}
906
907
908	/**
909	* Sets up and activates VT-x on the current CPU.
910	*
911	* @returns VBox status code.
912	* @param pCpu Pointer to the global CPU info struct.
913	* @param pVM Pointer to the VM (can be NULL after a host resume
914	* operation).
915	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
916	* fEnabledByHost is true).
917	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
918	* @a fEnabledByHost is true).
919	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
920	* enable VT-x/AMD-V on the host.
921	*/
922	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
923	{
924	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
925	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
926
927	if (!fEnabledByHost)
928	{
929	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
930	if (RT_FAILURE(rc))
931	return rc;
932	}
933
934	/*
935	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
936	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
937	* each time while reusing a VPID after hitting the MaxASID limit once.
938	*/
939	if ( pVM
940	&& pVM->hm.s.vmx.fVpid
941	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
942	{
943	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
944	pCpu->fFlushAsidBeforeUse = false;
945	}
946	else
947	pCpu->fFlushAsidBeforeUse = true;
948
949	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
950	++pCpu->cTlbFlushes;
951
952	return VINF_SUCCESS;
953	}
954
955
956	/**
957	* Deactivates VT-x on the current CPU.
958	*
959	* @returns VBox status code.
960	* @param pCpu Pointer to the global CPU info struct.
961	* @param pvCpuPage Pointer to the VMXON region.
962	* @param HCPhysCpuPage Physical address of the VMXON region.
963	*/
964	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
965	{
966	NOREF(pCpu);
967	NOREF(pvCpuPage);
968	NOREF(HCPhysCpuPage);
969
970	hmR0VmxLeaveRootMode();
971	return VINF_SUCCESS;
972	}
973
974
975	/**
976	* Sets the permission bits for the specified MSR in the MSR bitmap.
977	*
978	* @param pVCpu Pointer to the VMCPU.
979	* @param uMSR The MSR value.
980	* @param enmRead Whether reading this MSR causes a VM-exit.
981	* @param enmWrite Whether writing this MSR causes a VM-exit.
982	*/
983	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
984	{
985	int32_t iBit;
986	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
987
988	/*
989	* Layout:
990	* 0x000 - 0x3ff - Low MSR read bits
991	* 0x400 - 0x7ff - High MSR read bits
992	* 0x800 - 0xbff - Low MSR write bits
993	* 0xc00 - 0xfff - High MSR write bits
994	*/
995	if (uMsr <= 0x00001FFF)
996	iBit = uMsr;
997	else if ( uMsr >= 0xC0000000
998	&& uMsr <= 0xC0001FFF)
999	{
1000	iBit = (uMsr - 0xC0000000);
1001	pbMsrBitmap += 0x400;
1002	}
1003	else
1004	{
1005	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1006	return;
1007	}
1008
1009	Assert(iBit <= 0x1fff);
1010	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1011	ASMBitSet(pbMsrBitmap, iBit);
1012	else
1013	ASMBitClear(pbMsrBitmap, iBit);
1014
1015	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1016	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1017	else
1018	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1019	}
1020
1021
1022	/**
1023	* Flushes the TLB using EPT.
1024	*
1025	* @returns VBox status code.
1026	* @param pVM Pointer to the VM.
1027	* @param pVCpu Pointer to the VMCPU.
1028	* @param enmFlush Type of flush.
1029	*/
1030	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.fNestedPaging);
1034
1035	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1036
1037	uint64_t descriptor[2];
1038	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1039	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1040
1041	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1042	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1044	}
1045
1046
1047	/**
1048	* Flushes the TLB using VPID.
1049	*
1050	* @returns VBox status code.
1051	* @param pVM Pointer to the VM.
1052	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1053	* enmFlush).
1054	* @param enmFlush Type of flush.
1055	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1056	* on @a enmFlush).
1057	*/
1058	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1059	{
1060	AssertPtr(pVM);
1061	Assert(pVM->hm.s.vmx.fVpid);
1062
1063	uint64_t descriptor[2];
1064	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1065	{
1066	descriptor[0] = 0;
1067	descriptor[1] = 0;
1068	}
1069	else
1070	{
1071	AssertPtr(pVCpu);
1072	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1073	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1074	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1075	descriptor[1] = GCPtr;
1076	}
1077
1078	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1079	AssertMsg(rc == VINF_SUCCESS,
1080	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1081	if ( RT_SUCCESS(rc)
1082	&& pVCpu)
1083	{
1084	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1085	}
1086	}
1087
1088
1089	/**
1090	* Invalidates a guest page by guest virtual address. Only relevant for
1091	* EPT/VPID, otherwise there is nothing really to invalidate.
1092	*
1093	* @returns VBox status code.
1094	* @param pVM Pointer to the VM.
1095	* @param pVCpu Pointer to the VMCPU.
1096	* @param GCVirt Guest virtual address of the page to invalidate.
1097	*/
1098	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1099	{
1100	AssertPtr(pVM);
1101	AssertPtr(pVCpu);
1102	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1103
1104	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1105	if (!fFlushPending)
1106	{
1107	/*
1108	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1109	* See @bugref{6043} and @bugref{6177}.
1110	*
1111	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1112	* function maybe called in a loop with individual addresses.
1113	*/
1114	if (pVM->hm.s.vmx.fVpid)
1115	{
1116	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1117	{
1118	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1120	}
1121	else
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	}
1124	else if (pVM->hm.s.fNestedPaging)
1125	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1126	}
1127
1128	return VINF_SUCCESS;
1129	}
1130
1131
1132	/**
1133	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1134	* otherwise there is nothing really to invalidate.
1135	*
1136	* @returns VBox status code.
1137	* @param pVM Pointer to the VM.
1138	* @param pVCpu Pointer to the VMCPU.
1139	* @param GCPhys Guest physical address of the page to invalidate.
1140	*/
1141	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1142	{
1143	LogFlowFunc(("%RGp\n", GCPhys));
1144
1145	/*
1146	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1147	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1148	* This function might be called in a loop.
1149	*/
1150	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1151	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1152	return VINF_SUCCESS;
1153	}
1154
1155
1156	/**
1157	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1158	* case where neither EPT nor VPID is supported by the CPU.
1159	*
1160	* @param pVM Pointer to the VM.
1161	* @param pVCpu Pointer to the VMCPU.
1162	*
1163	* @remarks Called with interrupts disabled.
1164	*/
1165	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1166	{
1167	NOREF(pVM);
1168	AssertPtr(pVCpu);
1169	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1170	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1171
1172	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1173	AssertPtr(pCpu);
1174
1175	pVCpu->hm.s.TlbShootdown.cPages = 0;
1176	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1178	pVCpu->hm.s.fForceTLBFlush = false;
1179	return;
1180	}
1181
1182
1183	/**
1184	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1185	*
1186	* @param pVM Pointer to the VM.
1187	* @param pVCpu Pointer to the VMCPU.
1188	* @remarks All references to "ASID" in this function pertains to "VPID" in
1189	* Intel's nomenclature. The reason is, to avoid confusion in compare
1190	* statements since the host-CPU copies are named "ASID".
1191	*
1192	* @remarks Called with interrupts disabled.
1193	*/
1194	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1195	{
1196	AssertPtr(pVM);
1197	AssertPtr(pVCpu);
1198	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1199	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1200	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1201
1202	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1203	AssertPtr(pCpu);
1204
1205	/*
1206	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1207	* This can happen both for start & resume due to long jumps back to ring-3.
1208	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1209	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1210	*/
1211	bool fNewASID = false;
1212	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1213	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1214	{
1215	pVCpu->hm.s.fForceTLBFlush = true;
1216	fNewASID = true;
1217	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1218	}
1219
1220	/*
1221	* Check for explicit TLB shootdowns.
1222	*/
1223	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1224	{
1225	pVCpu->hm.s.fForceTLBFlush = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1227	}
1228
1229	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1230	if (pVCpu->hm.s.fForceTLBFlush)
1231	{
1232	if (fNewASID)
1233	{
1234	++pCpu->uCurrentAsid;
1235	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1236	{
1237	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1238	pCpu->cTlbFlushes++;
1239	pCpu->fFlushAsidBeforeUse = true;
1240	}
1241
1242	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1243	if (pCpu->fFlushAsidBeforeUse)
1244	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1245	}
1246	else
1247	{
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1249	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1250	else
1251	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1252	}
1253
1254	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1255	pVCpu->hm.s.fForceTLBFlush = false;
1256	}
1257	else
1258	{
1259	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1260	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1261	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1262	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1263
1264	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1265	* not be executed. See hmQueueInvlPage() where it is commented
1266	* out. Support individual entry flushing someday. */
1267	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1268	{
1269	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1270
1271	/*
1272	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1273	* as supported by the CPU.
1274	*/
1275	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1276	{
1277	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1278	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1279	}
1280	else
1281	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1282	}
1283	else
1284	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1285	}
1286	pVCpu->hm.s.TlbShootdown.cPages = 0;
1287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1288
1289	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1290	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1291	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1292	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1293	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1294	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1295
1296	/* Update VMCS with the VPID. */
1297	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1298	AssertRC(rc);
1299	}
1300
1301
1302	/**
1303	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1304	*
1305	* @returns VBox status code.
1306	* @param pVM Pointer to the VM.
1307	* @param pVCpu Pointer to the VMCPU.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1312	{
1313	AssertPtr(pVM);
1314	AssertPtr(pVCpu);
1315	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1316	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1317
1318	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1319	AssertPtr(pCpu);
1320
1321	/*
1322	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1323	* This can happen both for start & resume due to long jumps back to ring-3.
1324	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1325	*/
1326	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1327	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1328	{
1329	pVCpu->hm.s.fForceTLBFlush = true;
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1331	}
1332
1333	/* Check for explicit TLB shootdown flushes. */
1334	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1335	{
1336	pVCpu->hm.s.fForceTLBFlush = true;
1337	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1338	}
1339
1340	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343	if (pVCpu->hm.s.fForceTLBFlush)
1344	{
1345	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1346	pVCpu->hm.s.fForceTLBFlush = false;
1347	}
1348	else
1349	{
1350	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1351	* not be executed. See hmQueueInvlPage() where it is commented
1352	* out. Support individual entry flushing someday. */
1353	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1354	{
1355	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1357	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1358	}
1359	else
1360	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1361	}
1362
1363	pVCpu->hm.s.TlbShootdown.cPages = 0;
1364	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1365	}
1366
1367
1368	/**
1369	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1370	*
1371	* @returns VBox status code.
1372	* @param pVM Pointer to the VM.
1373	* @param pVCpu Pointer to the VMCPU.
1374	*
1375	* @remarks Called with interrupts disabled.
1376	*/
1377	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1378	{
1379	AssertPtr(pVM);
1380	AssertPtr(pVCpu);
1381	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1382	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1383
1384	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1385
1386	/*
1387	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1388	* This can happen both for start & resume due to long jumps back to ring-3.
1389	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1390	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1391	*/
1392	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1393	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1394	{
1395	pVCpu->hm.s.fForceTLBFlush = true;
1396	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1397	}
1398
1399	/* Check for explicit TLB shootdown flushes. */
1400	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1401	{
1402	/*
1403	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1404	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1405	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1406	*/
1407	pVCpu->hm.s.fForceTLBFlush = true;
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1409	}
1410
1411	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1412	if (pVCpu->hm.s.fForceTLBFlush)
1413	{
1414	++pCpu->uCurrentAsid;
1415	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1416	{
1417	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1418	pCpu->fFlushAsidBeforeUse = true;
1419	pCpu->cTlbFlushes++;
1420	}
1421
1422	pVCpu->hm.s.fForceTLBFlush = false;
1423	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1424	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1425	if (pCpu->fFlushAsidBeforeUse)
1426	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1427	}
1428	else
1429	{
1430	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1431	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1432	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1433	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1434
1435	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1436	* not be executed. See hmQueueInvlPage() where it is commented
1437	* out. Support individual entry flushing someday. */
1438	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1439	{
1440	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1441	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1442	{
1443	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1444	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1445	}
1446	else
1447	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1448	}
1449	else
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1451	}
1452
1453	pVCpu->hm.s.TlbShootdown.cPages = 0;
1454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1455
1456	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1457	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1458	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1459	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1460	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1461	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1462
1463	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1464	AssertRC(rc);
1465	}
1466
1467
1468	/**
1469	* Flushes the guest TLB entry based on CPU capabilities.
1470	*
1471	* @param pVCpu Pointer to the VMCPU.
1472	*/
1473	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1474	{
1475	PVM pVM = pVCpu->CTX_SUFF(pVM);
1476	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1477	{
1478	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1479	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1480	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1481	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1482	default:
1483	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1484	break;
1485	}
1486	}
1487
1488
1489	/**
1490	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1491	* TLB entries from the host TLB before VM-entry.
1492	*
1493	* @returns VBox status code.
1494	* @param pVM Pointer to the VM.
1495	*/
1496	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1497	{
1498	/*
1499	* Determine optimal flush type for nested paging.
1500	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1501	* guest execution (see hmR3InitFinalizeR0()).
1502	*/
1503	if (pVM->hm.s.fNestedPaging)
1504	{
1505	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1506	{
1507	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1508	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1509	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1510	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1511	else
1512	{
1513	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1514	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1515	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1516	}
1517
1518	/* Make sure the write-back cacheable memory type for EPT is supported. */
1519	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1520	{
1521	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1522	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1523	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1524	}
1525	}
1526	else
1527	{
1528	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1529	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1531	}
1532	}
1533
1534	/*
1535	* Determine optimal flush type for VPID.
1536	*/
1537	if (pVM->hm.s.vmx.fVpid)
1538	{
1539	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1540	{
1541	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1542	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1543	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1544	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1545	else
1546	{
1547	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1549	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1551	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1552	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1553	pVM->hm.s.vmx.fVpid = false;
1554	}
1555	}
1556	else
1557	{
1558	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1559	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1560	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1561	pVM->hm.s.vmx.fVpid = false;
1562	}
1563	}
1564
1565	/*
1566	* Setup the handler for flushing tagged-TLBs.
1567	*/
1568	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1569	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1570	else if (pVM->hm.s.fNestedPaging)
1571	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1572	else if (pVM->hm.s.vmx.fVpid)
1573	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1574	else
1575	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1576	return VINF_SUCCESS;
1577	}
1578
1579
1580	/**
1581	* Sets up pin-based VM-execution controls in the VMCS.
1582	*
1583	* @returns VBox status code.
1584	* @param pVM Pointer to the VM.
1585	* @param pVCpu Pointer to the VMCPU.
1586	*/
1587	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1588	{
1589	AssertPtr(pVM);
1590	AssertPtr(pVCpu);
1591
1592	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1593	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1594
1595	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1596	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1597	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1598
1599	/* Enable the VMX preemption timer. */
1600	if (pVM->hm.s.vmx.fUsePreemptTimer)
1601	{
1602	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1603	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1604	}
1605
1606	if ((val & zap) != val)
1607	{
1608	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1609	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1610	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1611	}
1612
1613	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1614	AssertRCReturn(rc, rc);
1615
1616	/* Update VCPU with the currently set pin-based VM-execution controls. */
1617	pVCpu->hm.s.vmx.u32PinCtls = val;
1618	return rc;
1619	}
1620
1621
1622	/**
1623	* Sets up processor-based VM-execution controls in the VMCS.
1624	*
1625	* @returns VBox status code.
1626	* @param pVM Pointer to the VM.
1627	* @param pVMCPU Pointer to the VMCPU.
1628	*/
1629	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1630	{
1631	AssertPtr(pVM);
1632	AssertPtr(pVCpu);
1633
1634	int rc = VERR_INTERNAL_ERROR_5;
1635	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1636	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1637
1638	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1639	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1640	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1641	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1645
1646	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1647	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1648	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1649	{
1650	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1652	}
1653
1654	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1655	if (!pVM->hm.s.fNestedPaging)
1656	{
1657	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1658	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1659	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1660	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1661	}
1662
1663	/* Use TPR shadowing if supported by the CPU. */
1664	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1665	{
1666	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1667	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1668	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1669	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1670	AssertRCReturn(rc, rc);
1671
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1673	/* CR8 writes causes a VM-exit based on TPR threshold. */
1674	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1675	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1676	}
1677	else
1678	{
1679	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1681	}
1682
1683	/* Use MSR-bitmaps if supported by the CPU. */
1684	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1685	{
1686	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1687
1688	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1689	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1690	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1691	AssertRCReturn(rc, rc);
1692
1693	/*
1694	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1695	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1696	*/
1697	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1698	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1699	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1700	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1701	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1702	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1703	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1704	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1705	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1706	}
1707
1708	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1710	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1715	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls = val;
1724
1725	/*
1726	* Secondary processor-based VM-execution controls.
1727	*/
1728	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1729	{
1730	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1731	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1732
1733	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1734	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1735
1736	if (pVM->hm.s.fNestedPaging)
1737	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1738	else
1739	{
1740	/*
1741	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1742	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1743	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1744	*/
1745	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1747	}
1748
1749	if (pVM->hm.s.vmx.fVpid)
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1751
1752	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1753	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1754
1755	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1756	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1757	* done dynamically. */
1758	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1759	{
1760	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1761	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1763	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1764	AssertRCReturn(rc, rc);
1765	}
1766
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1768	{
1769	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1771	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1772	}
1773
1774	if ((val & zap) != val)
1775	{
1776	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1777	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1778	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1779	}
1780
1781	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1782	AssertRCReturn(rc, rc);
1783
1784	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1785	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1786	}
1787
1788	return VINF_SUCCESS;
1789	}
1790
1791
1792	/**
1793	* Sets up miscellaneous (everything other than Pin & Processor-based
1794	* VM-execution) control fields in the VMCS.
1795	*
1796	* @returns VBox status code.
1797	* @param pVM Pointer to the VM.
1798	* @param pVCpu Pointer to the VMCPU.
1799	*/
1800	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1801	{
1802	AssertPtr(pVM);
1803	AssertPtr(pVCpu);
1804
1805	int rc = VERR_GENERAL_FAILURE;
1806
1807	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1808	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1809
1810	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1811
1812	/*
1813	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1814	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1815	* We thus use the exception bitmap to control it rather than use both.
1816	*/
1817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1818	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1819
1820	/** @todo Explore possibility of using IO-bitmaps. */
1821	/* All IO & IOIO instructions cause VM-exits. */
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1823	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1824
1825	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1826	/* Setup MSR autoloading/autostoring. */
1827	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1828	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1830	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1831	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1832	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1833
1834	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1835	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1836	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1837	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1838	#else
1839	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1840	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1841	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1842	#endif
1843
1844	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1845	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1846
1847	/* Setup debug controls */
1848	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1849	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1850	AssertRCReturn(rc, rc);
1851	return rc;
1852	}
1853
1854
1855	/**
1856	* Sets up the initial exception bitmap in the VMCS based on static conditions
1857	* (i.e. conditions that cannot ever change at runtime).
1858	*
1859	* @returns VBox status code.
1860	* @param pVM Pointer to the VM.
1861	* @param pVCpu Pointer to the VMCPU.
1862	*/
1863	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1864	{
1865	AssertPtr(pVM);
1866	AssertPtr(pVCpu);
1867
1868	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1869
1870	uint32_t u32XcptBitmap = 0;
1871
1872	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1873	if (!pVM->hm.s.fNestedPaging)
1874	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1875
1876	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1877	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1878	AssertRCReturn(rc, rc);
1879	return rc;
1880	}
1881
1882
1883	/**
1884	* Sets up the initial guest-state mask. The guest-state mask is consulted
1885	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1886	* for the nested virtualization case (as it would cause a VM-exit).
1887	*
1888	* @param pVCpu Pointer to the VMCPU.
1889	*/
1890	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1891	{
1892	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1893	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1894	return VINF_SUCCESS;
1895	}
1896
1897
1898	/**
1899	* Does per-VM VT-x initialization.
1900	*
1901	* @returns VBox status code.
1902	* @param pVM Pointer to the VM.
1903	*/
1904	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1905	{
1906	LogFlowFunc(("pVM=%p\n", pVM));
1907
1908	int rc = hmR0VmxStructsAlloc(pVM);
1909	if (RT_FAILURE(rc))
1910	{
1911	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1912	return rc;
1913	}
1914
1915	return VINF_SUCCESS;
1916	}
1917
1918
1919	/**
1920	* Does per-VM VT-x termination.
1921	*
1922	* @returns VBox status code.
1923	* @param pVM Pointer to the VM.
1924	*/
1925	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1926	{
1927	LogFlowFunc(("pVM=%p\n", pVM));
1928
1929	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1930	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1931	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1932	#endif
1933	hmR0VmxStructsFree(pVM);
1934	return VINF_SUCCESS;
1935	}
1936
1937
1938	/**
1939	* Sets up the VM for execution under VT-x.
1940	* This function is only called once per-VM during initalization.
1941	*
1942	* @returns VBox status code.
1943	* @param pVM Pointer to the VM.
1944	*/
1945	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1946	{
1947	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1948	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1949
1950	LogFlowFunc(("pVM=%p\n", pVM));
1951
1952	/*
1953	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1954	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1955	*/
1956	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1957	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1958	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1959	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1960	{
1961	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1962	return VERR_INTERNAL_ERROR;
1963	}
1964
1965	/* Initialize these always, see hmR3InitFinalizeR0().*/
1966	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1967	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1968
1969	/* Setup the tagged-TLB flush handlers. */
1970	int rc = hmR0VmxSetupTaggedTlb(pVM);
1971	if (RT_FAILURE(rc))
1972	{
1973	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1974	return rc;
1975	}
1976
1977	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1978	{
1979	PVMCPU pVCpu = &pVM->aCpus[i];
1980	AssertPtr(pVCpu);
1981	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1982
1983	/* Set revision dword at the beginning of the VMCS structure. */
1984	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1985
1986	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1987	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1988	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1989	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1990
1991	/* Load this VMCS as the current VMCS. */
1992	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1993	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1994	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1995
1996	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1997	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1998	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1999
2000	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2001	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2002	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2003
2004	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2017	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2018	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2019	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2020	#endif
2021
2022	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2023	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2024	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2025	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2026
2027	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2028	}
2029
2030	return VINF_SUCCESS;
2031	}
2032
2033
2034	/**
2035	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2036	* the VMCS.
2037	*
2038	* @returns VBox status code.
2039	* @param pVM Pointer to the VM.
2040	* @param pVCpu Pointer to the VMCPU.
2041	*/
2042	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2043	{
2044	RTCCUINTREG uReg = ASMGetCR0();
2045	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2046
2047	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2048	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2049	if (HMVMX_IS_64BIT_HOST_MODE())
2050	{
2051	uint64_t uRegCR3 = hmR0Get64bitCR3();
2052	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2053	}
2054	else
2055	#endif
2056	{
2057	uReg = ASMGetCR3();
2058	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2059	}
2060
2061	uReg = ASMGetCR4();
2062	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2063	AssertRCReturn(rc, rc);
2064	return rc;
2065	}
2066
2067
2068	/**
2069	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2070	* the host-state area in the VMCS.
2071	*
2072	* @returns VBox status code.
2073	* @param pVM Pointer to the VM.
2074	* @param pVCpu Pointer to the VMCPU.
2075	*/
2076	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2077	{
2078	int rc = VERR_INTERNAL_ERROR_5;
2079	RTSEL uSelCS = 0;
2080	RTSEL uSelSS = 0;
2081	RTSEL uSelDS = 0;
2082	RTSEL uSelES = 0;
2083	RTSEL uSelFS = 0;
2084	RTSEL uSelGS = 0;
2085	RTSEL uSelTR = 0;
2086
2087	/*
2088	* Host Selector registers.
2089	*/
2090	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2091	if (HMVMX_IS_64BIT_HOST_MODE())
2092	{
2093	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2094	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2095	}
2096	else
2097	{
2098	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2099	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2100	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2101	}
2102	#else
2103	uSelCS = ASMGetCS();
2104	uSelSS = ASMGetSS();
2105	#endif
2106
2107	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2108	uSelTR = ASMGetTR();
2109
2110	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2111	/** @todo Verify if we have any platform that actually run with DS or ES with
2112	* RPL != 0 in kernel space. */
2113	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2114	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2115	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2116	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2117	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2118	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2119	Assert(uSelCS != 0);
2120	Assert(uSelTR != 0);
2121
2122	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2123	#if 0
2124	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2125	Assert(uSelSS != 0);
2126	#endif
2127
2128	/* Write these host selector fields into the host-state area in the VMCS. */
2129	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2130	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2131	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2132	#if 0
2133	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2134	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2135	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2136	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2137	#endif
2138	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2139	AssertRCReturn(rc, rc);
2140
2141	/*
2142	* Host GDTR and IDTR.
2143	*/
2144	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2145	* be safe to -not- save and restore GDTR and IDTR in the assembly
2146	* code and just do it here and don't care if the limits are zapped on
2147	* VM-exit. */
2148	RTGDTR Gdtr;
2149	RT_ZERO(Gdtr);
2150	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2151	if (HMVMX_IS_64BIT_HOST_MODE())
2152	{
2153	X86XDTR64 Gdtr64;
2154	X86XDTR64 Idtr64;
2155	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2156	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2157	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2158	Gdtr.cbGdt = Gdtr64.cb;
2159	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2160	}
2161	else
2162	#endif
2163	{
2164	RTIDTR Idtr;
2165	ASMGetGDTR(&Gdtr);
2166	ASMGetIDTR(&Idtr);
2167	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2168	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2169	}
2170	AssertRCReturn(rc, rc);
2171
2172	/*
2173	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2174	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2175	*/
2176	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2177	{
2178	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2179	return VERR_VMX_INVALID_HOST_STATE;
2180	}
2181
2182	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2183	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2184	if (HMVMX_IS_64BIT_HOST_MODE())
2185	{
2186	/* We need the 64-bit TR base for hybrid darwin. */
2187	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2189	}
2190	else
2191	#endif
2192	{
2193	uintptr_t uTRBase;
2194	#if HC_ARCH_BITS == 64
2195	uTRBase = X86DESC64_BASE(pDesc);
2196	#else
2197	uTRBase = X86DESC_BASE(pDesc);
2198	#endif
2199	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2200	}
2201	AssertRCReturn(rc, rc);
2202
2203	/*
2204	* Host FS base and GS base.
2205	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2206	* would take care of the bases. In 64-bit, the MSRs come into play.
2207	*/
2208	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2209	if (HMVMX_IS_64BIT_HOST_MODE())
2210	{
2211	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2212	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2213	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2214	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2215	AssertRCReturn(rc, rc);
2216	}
2217	#endif
2218	return rc;
2219	}
2220
2221
2222	/**
2223	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2224	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2225	* the host after every successful VM exit.
2226	*
2227	* @returns VBox status code.
2228	* @param pVM Pointer to the VM.
2229	* @param pVCpu Pointer to the VMCPU.
2230	*/
2231	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2232	{
2233	AssertPtr(pVCpu);
2234	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2235
2236	int rc = VINF_SUCCESS;
2237	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2238	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2239	uint32_t cHostMsrs = 0;
2240	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2241
2242	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2243	{
2244	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2245	pHostMsr->u32Reserved = 0;
2246	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2247	if (CPUMIsGuestInLongMode(pVCpu))
2248	{
2249	/* Must match the EFER value in our 64 bits switcher. */
2250	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2251	}
2252	else
2253	# endif
2254	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2255	pHostMsr++; cHostMsrs++;
2256	}
2257
2258	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2259	if (HMVMX_IS_64BIT_HOST_MODE())
2260	{
2261	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2262	pHostMsr->u32Reserved = 0;
2263	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2264	pHostMsr++; cHostMsrs++;
2265	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2266	pHostMsr->u32Reserved = 0;
2267	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2268	pHostMsr++; cHostMsrs++;
2269	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2270	pHostMsr->u32Reserved = 0;
2271	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2272	pHostMsr++; cHostMsrs++;
2273	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2274	pHostMsr->u32Reserved = 0;
2275	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2276	pHostMsr++; cHostMsrs++;
2277	}
2278	# endif
2279
2280	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2281	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2282	{
2283	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2284	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2285	}
2286
2287	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2288	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2289
2290	/*
2291	* Host Sysenter MSRs.
2292	*/
2293	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2294	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2295	if (HMVMX_IS_64BIT_HOST_MODE())
2296	{
2297	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2298	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2299	}
2300	else
2301	{
2302	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2303	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2304	}
2305	# elif HC_ARCH_BITS == 32
2306	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2308	# else
2309	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2311	# endif
2312	AssertRCReturn(rc, rc);
2313
2314	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2315	* hmR0VmxSetupExitCtls() !! */
2316	return rc;
2317	}
2318
2319
2320	/**
2321	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2322	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2323	* controls".
2324	*
2325	* @returns VBox status code.
2326	* @param pVCpu Pointer to the VMCPU.
2327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2328	* out-of-sync. Make sure to update the required fields
2329	* before using them.
2330	*
2331	* @remarks No-long-jump zone!!!
2332	*/
2333	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2334	{
2335	int rc = VINF_SUCCESS;
2336	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2337	{
2338	PVM pVM = pVCpu->CTX_SUFF(pVM);
2339	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2340	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2341
2342	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2343	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2344
2345	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2346	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2347	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2348	else
2349	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2350
2351	/*
2352	* The following should not be set (since we're not in SMM mode):
2353	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2354	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2355	*/
2356
2357	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2358	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2359	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2360
2361	if ((val & zap) != val)
2362	{
2363	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2364	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2365	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2366	}
2367
2368	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2369	AssertRCReturn(rc, rc);
2370
2371	/* Update VCPU with the currently set VM-exit controls. */
2372	pVCpu->hm.s.vmx.u32EntryCtls = val;
2373	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2374	}
2375	return rc;
2376	}
2377
2378
2379	/**
2380	* Sets up the VM-exit controls in the VMCS.
2381	*
2382	* @returns VBox status code.
2383	* @param pVM Pointer to the VM.
2384	* @param pVCpu Pointer to the VMCPU.
2385	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2386	* out-of-sync. Make sure to update the required fields
2387	* before using them.
2388	*
2389	* @remarks requires EFER.
2390	*/
2391	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2392	{
2393	int rc = VINF_SUCCESS;
2394	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2395	{
2396	PVM pVM = pVCpu->CTX_SUFF(pVM);
2397	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2398	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2399
2400	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2401	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2402
2403	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2404	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2405	if (HMVMX_IS_64BIT_HOST_MODE())
2406	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2407	else
2408	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2409	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2410	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2411	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2412	else
2413	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2414	#endif
2415
2416	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2417	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2418
2419	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2420	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2421	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2422	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2423	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2424
2425	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2426	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2427
2428	if ((val & zap) != val)
2429	{
2430	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2431	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2432	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2433	}
2434
2435	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2436	AssertRCReturn(rc, rc);
2437
2438	/* Update VCPU with the currently set VM-exit controls. */
2439	pVCpu->hm.s.vmx.u32ExitCtls = val;
2440	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2441	}
2442	return rc;
2443	}
2444
2445
2446	/**
2447	* Loads the guest APIC and related state.
2448	*
2449	* @returns VBox status code.
2450	* @param pVM Pointer to the VM.
2451	* @param pVCpu Pointer to the VMCPU.
2452	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2453	* out-of-sync. Make sure to update the required fields
2454	* before using them.
2455	*/
2456	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2457	{
2458	int rc = VINF_SUCCESS;
2459	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2460	{
2461	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2462	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2463	{
2464	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2465
2466	bool fPendingIntr = false;
2467	uint8_t u8GuestTpr = 0;
2468	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2469	AssertRCReturn(rc, rc);
2470
2471	/*
2472	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2473	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2474	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2475	* the interrupt when we VM-exit for other reasons.
2476	*/
2477	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2478	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2479	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2480	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2481
2482	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2483	AssertRCReturn(rc, rc);
2484
2485	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2486	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2487	{
2488	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2489	pMixedCtx->msrLSTAR = u8GuestTpr;
2490	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2491	{
2492	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2493	if (fPendingIntr)
2494	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2495	else
2496	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497	}
2498	}
2499	}
2500
2501	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2502	}
2503	return rc;
2504	}
2505
2506
2507	/**
2508	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2509	*
2510	* @returns
2511	* @param pVCpu Pointer to the VMCPU.
2512	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2513	* out-of-sync. Make sure to update the required fields
2514	* before using them.
2515	*
2516	* @remarks No-long-jump zone!!!
2517	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2518	*/
2519	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2520	{
2521	/*
2522	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2523	* inhibit interrupts or clear any existing interrupt-inhibition.
2524	*/
2525	uint32_t uIntrState = 0;
2526	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2527	{
2528	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2529	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2530	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2531	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2532	{
2533	/*
2534	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2535	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2536	*/
2537	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2538	}
2539	else if (pMixedCtx->eflags.Bits.u1IF)
2540	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2541	else
2542	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2543	}
2544	return uIntrState;
2545	}
2546
2547
2548	/**
2549	* Loads the guest's interruptibility-state into the guest-state area in the
2550	* VMCS.
2551	*
2552	* @returns VBox status code.
2553	* @param pVCpu Pointer to the VMCPU.
2554	* @param uIntrState The interruptibility-state to set.
2555	*/
2556	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2557	{
2558	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2559	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2560	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2561	AssertRCReturn(rc, rc);
2562	return rc;
2563	}
2564
2565
2566	/**
2567	* Loads the guest's RIP into the guest-state area in the VMCS.
2568	*
2569	* @returns VBox status code.
2570	* @param pVCpu Pointer to the VMCPU.
2571	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2572	* out-of-sync. Make sure to update the required fields
2573	* before using them.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	*/
2577	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2578	{
2579	int rc = VINF_SUCCESS;
2580	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2581	{
2582	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2583	AssertRCReturn(rc, rc);
2584	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2585	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2586	}
2587	return rc;
2588	}
2589
2590
2591	/**
2592	* Loads the guest's RSP into the guest-state area in the VMCS.
2593	*
2594	* @returns VBox status code.
2595	* @param pVCpu Pointer to the VMCPU.
2596	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2597	* out-of-sync. Make sure to update the required fields
2598	* before using them.
2599	*
2600	* @remarks No-long-jump zone!!!
2601	*/
2602	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2603	{
2604	int rc = VINF_SUCCESS;
2605	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2606	{
2607	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2608	AssertRCReturn(rc, rc);
2609	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2610	}
2611	return rc;
2612	}
2613
2614
2615	/**
2616	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2617	*
2618	* @returns VBox status code.
2619	* @param pVCpu Pointer to the VMCPU.
2620	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2621	* out-of-sync. Make sure to update the required fields
2622	* before using them.
2623	*
2624	* @remarks No-long-jump zone!!!
2625	*/
2626	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2627	{
2628	int rc = VINF_SUCCESS;
2629	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2630	{
2631	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2632	Let us assert it as such and use 32-bit VMWRITE. */
2633	Assert(!(pMixedCtx->rflags.u64 >> 32));
2634	X86EFLAGS uEFlags = pMixedCtx->eflags;
2635	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2636	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2637
2638	/*
2639	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2640	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2641	*/
2642	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2643	{
2644	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2645	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2646	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2647	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2648	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2649	}
2650
2651	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2652	AssertRCReturn(rc, rc);
2653
2654	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2656	}
2657	return rc;
2658	}
2659
2660
2661	/**
2662	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2663	*
2664	* @returns VBox status code.
2665	* @param pVCpu Pointer to the VMCPU.
2666	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2667	* out-of-sync. Make sure to update the required fields
2668	* before using them.
2669	*
2670	* @remarks No-long-jump zone!!!
2671	*/
2672	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2673	{
2674	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2675	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2676	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2677	return rc;
2678	}
2679
2680
2681	/**
2682	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2683	* in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVM Pointer to the VM.
2687	* @param pVCpu Pointer to the VMCPU.
2688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2689	* out-of-sync. Make sure to update the required fields
2690	* before using them.
2691	*
2692	* @remarks No-long-jump zone!!!
2693	*/
2694	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2695	{
2696	int rc = VINF_SUCCESS;
2697	PVM pVM = pVCpu->CTX_SUFF(pVM);
2698
2699	/*
2700	* Guest CR0.
2701	* Guest FPU.
2702	*/
2703	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2704	{
2705	Assert(!(pCtx->cr0 >> 32));
2706	uint32_t u32GuestCR0 = pCtx->cr0;
2707
2708	/* The guest's view (read access) of its CR0 is unblemished. */
2709	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2710	AssertRCReturn(rc, rc);
2711	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2712
2713	/* Setup VT-x's view of the guest CR0. */
2714	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2715	if (pVM->hm.s.fNestedPaging)
2716	{
2717	if (CPUMIsGuestPagingEnabledEx(pCtx))
2718	{
2719	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2720	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2721	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2722	}
2723	else
2724	{
2725	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2726	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2727	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2728	}
2729
2730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2731	AssertRCReturn(rc, rc);
2732	}
2733	else
2734	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2735
2736	/*
2737	* Guest FPU bits.
2738	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2739	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2740	*/
2741	u32GuestCR0 \|= X86_CR0_NE;
2742	bool fInterceptNM = false;
2743	if (CPUMIsGuestFPUStateActive(pVCpu))
2744	{
2745	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2746	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2747	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2748	}
2749	else
2750	{
2751	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2752	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2753	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2754	}
2755
2756	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2757	bool fInterceptMF = false;
2758	if (!(pCtx->cr0 & X86_CR0_NE))
2759	fInterceptMF = true;
2760
2761	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2762	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2763	{
2764	Assert(PDMVmmDevHeapIsEnabled(pVM));
2765	Assert(pVM->hm.s.vmx.pRealModeTSS);
2766	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2767	fInterceptNM = true;
2768	fInterceptMF = true;
2769	}
2770	else
2771	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2772
2773	if (fInterceptNM)
2774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2775	else
2776	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2777
2778	if (fInterceptMF)
2779	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2780	else
2781	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2782
2783	/* Additional intercepts for debugging, define these yourself explicitly. */
2784	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2786	\| RT_BIT(X86_XCPT_DB)
2787	\| RT_BIT(X86_XCPT_DE)
2788	\| RT_BIT(X86_XCPT_NM)
2789	\| RT_BIT(X86_XCPT_UD)
2790	\| RT_BIT(X86_XCPT_NP)
2791	\| RT_BIT(X86_XCPT_SS)
2792	\| RT_BIT(X86_XCPT_GP)
2793	\| RT_BIT(X86_XCPT_PF)
2794	\| RT_BIT(X86_XCPT_MF);
2795	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2796	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2797	#endif
2798
2799	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2800
2801	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2802	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2803	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2804	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2805	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2806	else
2807	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2808
2809	u32GuestCR0 \|= uSetCR0;
2810	u32GuestCR0 &= uZapCR0;
2811	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2812
2813	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2814	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2815	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2816	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2817
2818	/*
2819	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2820	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2821	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2822	*/
2823	uint64_t u64CR0Mask = 0;
2824	u64CR0Mask = X86_CR0_PE
2825	\| X86_CR0_NE
2826	\| X86_CR0_WP
2827	\| X86_CR0_PG
2828	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2829	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2830	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2831	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2832	u64CR0Mask &= ~X86_CR0_PE;
2833	if (pVM->hm.s.fNestedPaging)
2834	u64CR0Mask &= ~X86_CR0_WP;
2835
2836	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2837	if (fInterceptNM)
2838	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2839	else
2840	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2841
2842	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2843	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2844	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2845	AssertRCReturn(rc, rc);
2846
2847	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2848	}
2849
2850	/*
2851	* Guest CR2.
2852	* It's always loaded in the assembler code. Nothing to do here.
2853	*/
2854
2855	/*
2856	* Guest CR3.
2857	*/
2858	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2859	{
2860	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2861	if (pVM->hm.s.fNestedPaging)
2862	{
2863	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2864
2865	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2866	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2867	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2868	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2869
2870	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2871	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2872	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2873
2874	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2875	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2876	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2877	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2878
2879	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2880	AssertRCReturn(rc, rc);
2881	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2882
2883	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2884	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2885	{
2886	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2887	if (CPUMIsGuestInPAEModeEx(pCtx))
2888	{
2889	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2890	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2891	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2892	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2893	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2894	AssertRCReturn(rc, rc);
2895	}
2896
2897	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2898	have Unrestricted Execution to handle the guest when it's not using paging. */
2899	GCPhysGuestCR3 = pCtx->cr3;
2900	}
2901	else
2902	{
2903	/*
2904	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2905	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2906	* EPT takes care of translating it to host-physical addresses.
2907	*/
2908	RTGCPHYS GCPhys;
2909	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2910	Assert(PDMVmmDevHeapIsEnabled(pVM));
2911
2912	/* We obtain it here every time as the guest could have relocated this PCI region. */
2913	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2914	AssertRCReturn(rc, rc);
2915
2916	GCPhysGuestCR3 = GCPhys;
2917	}
2918	}
2919	else
2920	{
2921	/* Non-nested paging case, just use the hypervisor's CR3. */
2922	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2923	}
2924
2925	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2926	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2927	AssertRCReturn(rc, rc);
2928
2929	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2930	}
2931
2932	/*
2933	* Guest CR4.
2934	*/
2935	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2936	{
2937	Assert(!(pCtx->cr4 >> 32));
2938	uint32_t u32GuestCR4 = pCtx->cr4;
2939
2940	/* The guest's view of its CR4 is unblemished. */
2941	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2942	AssertRCReturn(rc, rc);
2943	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2944
2945	/* Setup VT-x's view of the guest CR4. */
2946	/*
2947	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2948	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2949	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2950	*/
2951	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2952	{
2953	Assert(pVM->hm.s.vmx.pRealModeTSS);
2954	Assert(PDMVmmDevHeapIsEnabled(pVM));
2955	u32GuestCR4 &= ~X86_CR4_VME;
2956	}
2957
2958	if (pVM->hm.s.fNestedPaging)
2959	{
2960	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2961	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2962	{
2963	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2964	u32GuestCR4 \|= X86_CR4_PSE;
2965	/* Our identity mapping is a 32 bits page directory. */
2966	u32GuestCR4 &= ~X86_CR4_PAE;
2967	}
2968	/* else use guest CR4.*/
2969	}
2970	else
2971	{
2972	/*
2973	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2974	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2975	*/
2976	switch (pVCpu->hm.s.enmShadowMode)
2977	{
2978	case PGMMODE_REAL: /* Real-mode. */
2979	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2980	case PGMMODE_32_BIT: /* 32-bit paging. */
2981	{
2982	u32GuestCR4 &= ~X86_CR4_PAE;
2983	break;
2984	}
2985
2986	case PGMMODE_PAE: /* PAE paging. */
2987	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2988	{
2989	u32GuestCR4 \|= X86_CR4_PAE;
2990	break;
2991	}
2992
2993	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2994	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2995	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2996	break;
2997	#endif
2998	default:
2999	AssertFailed();
3000	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3001	}
3002	}
3003
3004	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3005	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3006	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3007	u32GuestCR4 \|= uSetCR4;
3008	u32GuestCR4 &= uZapCR4;
3009
3010	/* Write VT-x's view of the guest CR4 into the VMCS. */
3011	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3012	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3013
3014	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3015	uint64_t u64CR4Mask = 0;
3016	u64CR4Mask = X86_CR4_VME
3017	\| X86_CR4_PAE
3018	\| X86_CR4_PGE
3019	\| X86_CR4_PSE
3020	\| X86_CR4_VMXE;
3021	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3022	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3023	AssertRCReturn(rc, rc);
3024
3025	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3026	}
3027	return rc;
3028	}
3029
3030
3031	/**
3032	* Loads the guest debug registers into the guest-state area in the VMCS.
3033	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3034	*
3035	* @returns VBox status code.
3036	* @param pVCpu Pointer to the VMCPU.
3037	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3038	* out-of-sync. Make sure to update the required fields
3039	* before using them.
3040	*
3041	* @remarks No-long-jump zone!!!
3042	*/
3043	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3044	{
3045	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3046	return VINF_SUCCESS;
3047
3048	#ifdef VBOX_STRICT
3049	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3050	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3051	{
3052	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3053	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3054	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3055	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3056	}
3057	#endif
3058
3059	int rc = VERR_INTERNAL_ERROR_5;
3060	PVM pVM = pVCpu->CTX_SUFF(pVM);
3061	bool fInterceptDB = false;
3062	bool fInterceptMovDRx = false;
3063	if (DBGFIsStepping(pVCpu))
3064	{
3065	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3066	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3067	{
3068	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3069	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3070	AssertRCReturn(rc, rc);
3071	Assert(fInterceptDB == false);
3072	}
3073	else
3074	fInterceptDB = true;
3075	}
3076
3077	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3078	{
3079	if (!CPUMIsHyperDebugStateActive(pVCpu))
3080	{
3081	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3082	AssertRC(rc);
3083	}
3084	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3085	fInterceptMovDRx = true;
3086	}
3087	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3088	{
3089	if (!CPUMIsGuestDebugStateActive(pVCpu))
3090	{
3091	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3092	AssertRC(rc);
3093	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3094	}
3095	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3096	Assert(fInterceptMovDRx == false);
3097	}
3098	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3099	{
3100	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3101	fInterceptMovDRx = true;
3102	}
3103
3104	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3105	if (fInterceptDB)
3106	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3107	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3108	{
3109	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3110	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3111	#endif
3112	}
3113
3114	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3115	if (fInterceptMovDRx)
3116	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3117	else
3118	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3119
3120	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3121	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3122
3123	/* The guest's view of its DR7 is unblemished. */
3124	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3125
3126	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3127	return rc;
3128	}
3129
3130
3131	#ifdef VBOX_STRICT
3132	/**
3133	* Strict function to validate segment registers.
3134	*/
3135	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3136	{
3137	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3138	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3139	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3140	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3141	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3142	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3143	{
3144	/* Protected mode checks */
3145	/* CS */
3146	Assert(pCtx->cs.Attr.n.u1Present);
3147	Assert(!(pCtx->cs.Attr.u & 0xf00));
3148	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3149	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3150	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3151	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3152	\|\| (pCtx->cs.Attr.n.u1Granularity));
3153	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3154	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3155	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3156	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3157	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3158	else
3159	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3160	/* SS */
3161	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3162	{
3163	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3164	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3165	Assert(pCtx->ss.Attr.n.u1Present);
3166	Assert(!(pCtx->ss.Attr.u & 0xf00));
3167	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3168	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3169	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3170	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3171	\|\| (pCtx->ss.Attr.n.u1Granularity));
3172	}
3173	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3174	/* CR0 might not be up-to-date here always, hence disabled. */
3175	#if 0
3176	if (!pCtx->cr0 & X86_CR0_PE)
3177	Assert(!pCtx->ss.Attr.n.u2Dpl);
3178	#endif
3179	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3180	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3181	{
3182	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3183	Assert(pCtx->ds.Attr.n.u1Present);
3184	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3185	Assert(!(pCtx->ds.Attr.u & 0xf00));
3186	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3187	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3188	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3189	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3190	\|\| (pCtx->ds.Attr.n.u1Granularity));
3191	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3192	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3193	}
3194	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3195	{
3196	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3197	Assert(pCtx->es.Attr.n.u1Present);
3198	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3199	Assert(!(pCtx->es.Attr.u & 0xf00));
3200	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3201	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3202	\|\| !(pCtx->es.Attr.n.u1Granularity));
3203	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3204	\|\| (pCtx->es.Attr.n.u1Granularity));
3205	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3206	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3207	}
3208	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3209	{
3210	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3211	Assert(pCtx->fs.Attr.n.u1Present);
3212	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3213	Assert(!(pCtx->fs.Attr.u & 0xf00));
3214	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3215	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3216	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3217	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3218	\|\| (pCtx->fs.Attr.n.u1Granularity));
3219	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3220	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3221	}
3222	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3223	{
3224	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3225	Assert(pCtx->gs.Attr.n.u1Present);
3226	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3227	Assert(!(pCtx->gs.Attr.u & 0xf00));
3228	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3229	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3230	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3231	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3232	\|\| (pCtx->gs.Attr.n.u1Granularity));
3233	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3234	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3235	}
3236	/* 64-bit capable CPUs. */
3237	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3238	Assert(!(pCtx->cs.u64Base >> 32));
3239	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3240	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3241	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3242	# endif
3243	}
3244	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3245	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3246	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3247	{
3248	/* Real and v86 mode checks. */
3249	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3250	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3251	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3252	{
3253	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3254	}
3255	else
3256	{
3257	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3258	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3259	}
3260
3261	/* CS */
3262	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3263	Assert(pCtx->cs.u32Limit == 0xffff);
3264	Assert(u32CSAttr == 0xf3);
3265	/* SS */
3266	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3267	Assert(pCtx->ss.u32Limit == 0xffff);
3268	Assert(u32SSAttr == 0xf3);
3269	/* DS */
3270	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3271	Assert(pCtx->ds.u32Limit == 0xffff);
3272	Assert(u32DSAttr == 0xf3);
3273	/* ES */
3274	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3275	Assert(pCtx->es.u32Limit == 0xffff);
3276	Assert(u32ESAttr == 0xf3);
3277	/* FS */
3278	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3279	Assert(pCtx->fs.u32Limit == 0xffff);
3280	Assert(u32FSAttr == 0xf3);
3281	/* GS */
3282	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3283	Assert(pCtx->gs.u32Limit == 0xffff);
3284	Assert(u32GSAttr == 0xf3);
3285	/* 64-bit capable CPUs. */
3286	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3287	Assert(!(pCtx->cs.u64Base >> 32));
3288	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3289	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3290	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3291	# endif
3292	}
3293	}
3294	#endif /* VBOX_STRICT */
3295
3296
3297	/**
3298	* Writes a guest segment register into the guest-state area in the VMCS.
3299	*
3300	* @returns VBox status code.
3301	* @param pVCpu Pointer to the VMCPU.
3302	* @param idxSel Index of the selector in the VMCS.
3303	* @param idxLimit Index of the segment limit in the VMCS.
3304	* @param idxBase Index of the segment base in the VMCS.
3305	* @param idxAccess Index of the access rights of the segment in the VMCS.
3306	* @param pSelReg Pointer to the segment selector.
3307	* @param pCtx Pointer to the guest-CPU context.
3308	*
3309	* @remarks No-long-jump zone!!!
3310	*/
3311	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3312	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3313	{
3314	int rc;
3315	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3316	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3317	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3318	AssertRCReturn(rc, rc);
3319
3320	uint32_t u32Access = pSelReg->Attr.u;
3321	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3322	{
3323	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3324	u32Access = 0xf3;
3325	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3326	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3327	}
3328	else
3329	{
3330	/*
3331	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3332	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3333	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3334	* loaded in protected-mode have their attribute as 0.
3335	*/
3336	if (!u32Access)
3337	u32Access = HMVMX_SEL_UNUSABLE;
3338	}
3339
3340	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3341	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3342	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3343
3344	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3345	AssertRCReturn(rc, rc);
3346	return rc;
3347	}
3348
3349
3350	/**
3351	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3352	* into the guest-state area in the VMCS.
3353	*
3354	* @returns VBox status code.
3355	* @param pVM Pointer to the VM.
3356	* @param pVCPU Pointer to the VMCPU.
3357	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3358	* out-of-sync. Make sure to update the required fields
3359	* before using them.
3360	*
3361	* @remarks No-long-jump zone!!!
3362	*/
3363	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3364	{
3365	int rc = VERR_INTERNAL_ERROR_5;
3366	PVM pVM = pVCpu->CTX_SUFF(pVM);
3367
3368	/*
3369	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3370	*/
3371	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3372	{
3373	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3374	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3375	{
3376	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3377	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3378	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3379	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3380	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3381	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3382	}
3383
3384	#ifdef VBOX_WITH_REM
3385	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3386	{
3387	Assert(pVM->hm.s.vmx.pRealModeTSS);
3388	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3389	if ( pVCpu->hm.s.vmx.fWasInRealMode
3390	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3391	{
3392	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3393	in real-mode (e.g. OpenBSD 4.0) */
3394	REMFlushTBs(pVM);
3395	Log(("Load: Switch to protected mode detected!\n"));
3396	pVCpu->hm.s.vmx.fWasInRealMode = false;
3397	}
3398	}
3399	#endif
3400	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3401	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3402	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3403	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3404	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3405	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3406	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3407	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3408	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3409	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3410	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3411	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3412	AssertRCReturn(rc, rc);
3413
3414	#ifdef VBOX_STRICT
3415	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3416	#endif
3417	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3418	}
3419
3420	/*
3421	* Guest TR.
3422	*/
3423	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3424	{
3425	/*
3426	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3427	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3428	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3429	*/
3430	uint16_t u16Sel = 0;
3431	uint32_t u32Limit = 0;
3432	uint64_t u64Base = 0;
3433	uint32_t u32AccessRights = 0;
3434
3435	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3436	{
3437	u16Sel = pMixedCtx->tr.Sel;
3438	u32Limit = pMixedCtx->tr.u32Limit;
3439	u64Base = pMixedCtx->tr.u64Base;
3440	u32AccessRights = pMixedCtx->tr.Attr.u;
3441	}
3442	else
3443	{
3444	Assert(pVM->hm.s.vmx.pRealModeTSS);
3445	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3446
3447	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3448	RTGCPHYS GCPhys;
3449	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3450	AssertRCReturn(rc, rc);
3451
3452	X86DESCATTR DescAttr;
3453	DescAttr.u = 0;
3454	DescAttr.n.u1Present = 1;
3455	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3456
3457	u16Sel = 0;
3458	u32Limit = HM_VTX_TSS_SIZE;
3459	u64Base = GCPhys; /* in real-mode phys = virt. */
3460	u32AccessRights = DescAttr.u;
3461	}
3462
3463	/* Validate. */
3464	Assert(!(u16Sel & RT_BIT(2)));
3465	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3466	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3467	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3468	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3469	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3470	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3471	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3472	Assert( (u32Limit & 0xfff) == 0xfff
3473	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3474	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3475	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3476
3477	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3478	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3479	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3480	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3481	AssertRCReturn(rc, rc);
3482
3483	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3484	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3485	}
3486
3487	/*
3488	* Guest GDTR.
3489	*/
3490	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3491	{
3492	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3493	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3494	AssertRCReturn(rc, rc);
3495
3496	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3497	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3498	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3499	}
3500
3501	/*
3502	* Guest LDTR.
3503	*/
3504	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3505	{
3506	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3507	uint32_t u32Access = 0;
3508	if (!pMixedCtx->ldtr.Attr.u)
3509	u32Access = HMVMX_SEL_UNUSABLE;
3510	else
3511	u32Access = pMixedCtx->ldtr.Attr.u;
3512
3513	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3514	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3515	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3516	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3517	AssertRCReturn(rc, rc);
3518
3519	/* Validate. */
3520	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3521	{
3522	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3523	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3524	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3525	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3526	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3527	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3528	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3529	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3530	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3531	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3532	}
3533
3534	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3535	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3536	}
3537
3538	/*
3539	* Guest IDTR.
3540	*/
3541	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3542	{
3543	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3544	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3545	AssertRCReturn(rc, rc);
3546
3547	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3548	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3549	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3550	}
3551
3552	return VINF_SUCCESS;
3553	}
3554
3555
3556	/**
3557	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3558	* areas. These MSRs will automatically be loaded to the host CPU on every
3559	* successful VM entry and stored from the host CPU on every successful VM exit.
3560	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3561	*
3562	* @returns VBox status code.
3563	* @param pVCpu Pointer to the VMCPU.
3564	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565	* out-of-sync. Make sure to update the required fields
3566	* before using them.
3567	*
3568	* @remarks No-long-jump zone!!!
3569	*/
3570	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3571	{
3572	AssertPtr(pVCpu);
3573	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3574
3575	/*
3576	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3577	*/
3578	int rc = VINF_SUCCESS;
3579	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3580	{
3581	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3582	PVM pVM = pVCpu->CTX_SUFF(pVM);
3583	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3584	uint32_t cGuestMsrs = 0;
3585
3586	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3587	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3588	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3589	if (fSupportsNX \|\| fSupportsLongMode)
3590	{
3591	/** @todo support save IA32_EFER, i.e.
3592	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3593	* guest EFER need not be part of the VM-entry MSR-load area. */
3594	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3595	pGuestMsr->u32Reserved = 0;
3596	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3597	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3598	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3599	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3600	pGuestMsr++; cGuestMsrs++;
3601	if (fSupportsLongMode)
3602	{
3603	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3604	pGuestMsr->u32Reserved = 0;
3605	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3606	pGuestMsr++; cGuestMsrs++;
3607	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3608	pGuestMsr->u32Reserved = 0;
3609	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3610	pGuestMsr++; cGuestMsrs++;
3611	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3612	pGuestMsr->u32Reserved = 0;
3613	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3614	pGuestMsr++; cGuestMsrs++;
3615	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3618	pGuestMsr++; cGuestMsrs++;
3619	}
3620	}
3621
3622	/*
3623	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3624	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3625	*/
3626	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3627	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3628	{
3629	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3630	pGuestMsr->u32Reserved = 0;
3631	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3632	AssertRCReturn(rc, rc);
3633	pGuestMsr++; cGuestMsrs++;
3634	}
3635
3636	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3637	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3638	{
3639	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3640	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3641	}
3642
3643	/* Update the VCPU's copy of the guest MSR count. */
3644	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3646	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3647	AssertRCReturn(rc, rc);
3648	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3649
3650	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3651	}
3652
3653	/*
3654	* Guest Sysenter MSRs.
3655	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3656	* VM-exits on WRMSRs for these MSRs.
3657	*/
3658	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3659	{
3660	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3661	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3662	}
3663	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3664	{
3665	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3666	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3667	}
3668	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3669	{
3670	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3671	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3672	}
3673	AssertRCReturn(rc, rc);
3674
3675	return rc;
3676	}
3677
3678
3679	/**
3680	* Loads the guest activity state into the guest-state area in the VMCS.
3681	*
3682	* @returns VBox status code.
3683	* @param pVCpu Pointer to the VMCPU.
3684	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685	* out-of-sync. Make sure to update the required fields
3686	* before using them.
3687	*
3688	* @remarks No-long-jump zone!!!
3689	*/
3690	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3691	{
3692	/** @todo See if we can make use of other states, e.g.
3693	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3694	int rc = VINF_SUCCESS;
3695	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3696	{
3697	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3698	AssertRCReturn(rc, rc);
3699	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3700	}
3701	return rc;
3702	}
3703
3704
3705	/**
3706	* Sets up the appropriate function to run guest code.
3707	*
3708	* @returns VBox status code.
3709	* @param pVCpu Pointer to the VMCPU.
3710	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3711	* out-of-sync. Make sure to update the required fields
3712	* before using them.
3713	*
3714	* @remarks No-long-jump zone!!!
3715	*/
3716	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3717	{
3718	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3719	{
3720	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3721	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3722	#endif
3723	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3724	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3725	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3726	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3727	#else
3728	/* 64-bit host or hybrid host. */
3729	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3730	#endif
3731	}
3732	else
3733	{
3734	/* Guest is not in long mode, use the 32-bit handler. */
3735	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3736	}
3737	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3738	return VINF_SUCCESS;
3739	}
3740
3741
3742	/**
3743	* Wrapper for running the guest code in VT-x.
3744	*
3745	* @returns VBox strict status code.
3746	* @param pVM Pointer to the VM.
3747	* @param pVCpu Pointer to the VMCPU.
3748	* @param pCtx Pointer to the guest-CPU context.
3749	*
3750	* @remarks No-long-jump zone!!!
3751	*/
3752	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3753	{
3754	/*
3755	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3756	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3757	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3758	*/
3759	#ifdef VBOX_WITH_KERNEL_USING_XMM
3760	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3761	#else
3762	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3763	#endif
3764	}
3765
3766
3767	/**
3768	* Report world-switch error and dump some useful debug info.
3769	*
3770	* @param pVM Pointer to the VM.
3771	* @param pVCpu Pointer to the VMCPU.
3772	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3773	* @param pCtx Pointer to the guest-CPU context.
3774	* @param pVmxTransient Pointer to the VMX transient structure (only
3775	* exitReason updated).
3776	*/
3777	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3778	{
3779	Assert(pVM);
3780	Assert(pVCpu);
3781	Assert(pCtx);
3782	Assert(pVmxTransient);
3783	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3784
3785	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3786	switch (rcVMRun)
3787	{
3788	case VERR_VMX_INVALID_VMXON_PTR:
3789	AssertFailed();
3790	break;
3791	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3792	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3793	{
3794	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3795	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3796	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3797	AssertRC(rc);
3798
3799	#ifdef VBOX_STRICT
3800	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3801	pVmxTransient->uExitReason));
3802	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3803	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3804	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3805	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3806	else
3807	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3808
3809	/* VMX control bits. */
3810	uint32_t u32Val;
3811	uint64_t u64Val;
3812	HMVMXHCUINTREG uHCReg;
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3845	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3846	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3847	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3848	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3849	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3850	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3852	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3853	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3854	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3855
3856	/* Guest bits. */
3857	RTGCUINTREG uGCReg;
3858	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3859	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3860	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3861	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3862	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3863	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3864	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3865	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3866
3867	/* Host bits. */
3868	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3869	Log(("Host CR0 %#RHr\n", uHCReg));
3870	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3871	Log(("Host CR3 %#RHr\n", uHCReg));
3872	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3873	Log(("Host CR4 %#RHr\n", uHCReg));
3874
3875	RTGDTR HostGdtr;
3876	PCX86DESCHC pDesc;
3877	ASMGetGDTR(&HostGdtr);
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3879	Log(("Host CS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3887	Log(("Host DS %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3892	}
3893
3894	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3895	Log(("Host ES %#08x\n", u32Val));
3896	if (u32Val < HostGdtr.cbGdt)
3897	{
3898	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3899	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3900	}
3901
3902	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3903	Log(("Host FS %#08x\n", u32Val));
3904	if (u32Val < HostGdtr.cbGdt)
3905	{
3906	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3907	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3908	}
3909
3910	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3911	Log(("Host GS %#08x\n", u32Val));
3912	if (u32Val < HostGdtr.cbGdt)
3913	{
3914	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3915	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3916	}
3917
3918	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3919	Log(("Host SS %#08x\n", u32Val));
3920	if (u32Val < HostGdtr.cbGdt)
3921	{
3922	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3923	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3924	}
3925
3926	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3927	Log(("Host TR %#08x\n", u32Val));
3928	if (u32Val < HostGdtr.cbGdt)
3929	{
3930	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3931	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3932	}
3933
3934	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3935	Log(("Host TR Base %#RHv\n", uHCReg));
3936	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3937	Log(("Host GDTR Base %#RHv\n", uHCReg));
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3939	Log(("Host IDTR Base %#RHv\n", uHCReg));
3940	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3941	Log(("Host SYSENTER CS %#08x\n", u32Val));
3942	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3943	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3944	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3945	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3947	Log(("Host RSP %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3949	Log(("Host RIP %#RHv\n", uHCReg));
3950	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3951	if (HMVMX_IS_64BIT_HOST_MODE())
3952	{
3953	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3954	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3955	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3956	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3957	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3958	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3959	}
3960	# endif
3961	#endif /* VBOX_STRICT */
3962	break;
3963	}
3964
3965	default:
3966	/* Impossible */
3967	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3968	break;
3969	}
3970	NOREF(pVM);
3971	}
3972
3973
3974	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3975	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3976	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3977	#endif
3978	#ifdef VBOX_STRICT
3979	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3980	{
3981	switch (idxField)
3982	{
3983	case VMX_VMCS_GUEST_RIP:
3984	case VMX_VMCS_GUEST_RSP:
3985	case VMX_VMCS_GUEST_DR7:
3986	case VMX_VMCS_GUEST_SYSENTER_EIP:
3987	case VMX_VMCS_GUEST_SYSENTER_ESP:
3988	case VMX_VMCS_GUEST_GDTR_BASE:
3989	case VMX_VMCS_GUEST_IDTR_BASE:
3990	case VMX_VMCS_GUEST_CS_BASE:
3991	case VMX_VMCS_GUEST_DS_BASE:
3992	case VMX_VMCS_GUEST_ES_BASE:
3993	case VMX_VMCS_GUEST_FS_BASE:
3994	case VMX_VMCS_GUEST_GS_BASE:
3995	case VMX_VMCS_GUEST_SS_BASE:
3996	case VMX_VMCS_GUEST_LDTR_BASE:
3997	case VMX_VMCS_GUEST_TR_BASE:
3998	case VMX_VMCS_GUEST_CR3:
3999	return true;
4000	}
4001	return false;
4002	}
4003
4004	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4005	{
4006	switch (idxField)
4007	{
4008	/* Read-only fields. */
4009	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4010	return true;
4011	}
4012	/* Remaining readable fields should also be writable. */
4013	return hmR0VmxIsValidWriteField(idxField);
4014	}
4015	#endif /* VBOX_STRICT */
4016
4017	/**
4018	* Executes the specified handler in 64-bit mode.
4019	*
4020	* @returns VBox status code.
4021	* @param pVM Pointer to the VM.
4022	* @param pVCpu Pointer to the VMCPU.
4023	* @param pCtx Pointer to the guest CPU context.
4024	* @param enmOp The operation to perform.
4025	* @param cbParam Number of parameters.
4026	* @param paParam Array of 32-bit parameters.
4027	*/
4028	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4029	uint32_t *paParam)
4030	{
4031	int rc, rc2;
4032	PHMGLOBLCPUINFO pCpu;
4033	RTHCPHYS HCPhysCpuPage;
4034	RTCCUINTREG uOldEFlags;
4035
4036	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4037	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4038	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4039	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4040
4041	#ifdef VBOX_STRICT
4042	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4043	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4044
4045	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4046	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4047	#endif
4048
4049	/* Disable interrupts. */
4050	uOldEFlags = ASMIntDisableFlags();
4051
4052	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4053	RTCPUID idHostCpu = RTMpCpuId();
4054	CPUMR0SetLApic(pVM, idHostCpu);
4055	#endif
4056
4057	pCpu = HMR0GetCurrentCpu();
4058	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4059
4060	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4061	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4062
4063	/* Leave VMX Root Mode. */
4064	VMXDisable();
4065
4066	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4067
4068	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4069	CPUMSetHyperEIP(pVCpu, enmOp);
4070	for (int i = (int)cbParam - 1; i >= 0; i--)
4071	CPUMPushHyper(pVCpu, paParam[i]);
4072
4073	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4074
4075	/* Call the switcher. */
4076	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4077	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4078
4079	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4080	/* Make sure the VMX instructions don't cause #UD faults. */
4081	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4082
4083	/* Re-enter VMX Root Mode */
4084	rc2 = VMXEnable(HCPhysCpuPage);
4085	if (RT_FAILURE(rc2))
4086	{
4087	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4088	ASMSetFlags(uOldEFlags);
4089	return rc2;
4090	}
4091
4092	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4093	AssertRC(rc2);
4094	Assert(!(ASMGetFlags() & X86_EFL_IF));
4095	ASMSetFlags(uOldEFlags);
4096	return rc;
4097	}
4098
4099
4100	/**
4101	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4102	* supporting 64-bit guests.
4103	*
4104	* @returns VBox status code.
4105	* @param fResume Whether to VMLAUNCH or VMRESUME.
4106	* @param pCtx Pointer to the guest-CPU context.
4107	* @param pCache Pointer to the VMCS cache.
4108	* @param pVM Pointer to the VM.
4109	* @param pVCpu Pointer to the VMCPU.
4110	*/
4111	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4112	{
4113	uint32_t aParam[6];
4114	PHMGLOBLCPUINFO pCpu = NULL;
4115	RTHCPHYS HCPhysCpuPage = 0;
4116	int rc = VERR_INTERNAL_ERROR_5;
4117
4118	pCpu = HMR0GetCurrentCpu();
4119	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4120
4121	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4122	pCache->uPos = 1;
4123	pCache->interPD = PGMGetInterPaeCR3(pVM);
4124	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4125	#endif
4126
4127	#ifdef VBOX_STRICT
4128	pCache->TestIn.HCPhysCpuPage = 0;
4129	pCache->TestIn.HCPhysVmcs = 0;
4130	pCache->TestIn.pCache = 0;
4131	pCache->TestOut.HCPhysVmcs = 0;
4132	pCache->TestOut.pCache = 0;
4133	pCache->TestOut.pCtx = 0;
4134	pCache->TestOut.eflags = 0;
4135	#endif
4136
4137	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4138	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4139	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4140	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4141	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4142	aParam[5] = 0;
4143
4144	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4145	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4146	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4147	#endif
4148	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4149
4150	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4151	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4152	Assert(pCtx->dr[4] == 10);
4153	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4154	#endif
4155
4156	#ifdef VBOX_STRICT
4157	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4158	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4159	pVCpu->hm.s.vmx.HCPhysVmcs));
4160	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4161	pCache->TestOut.HCPhysVmcs));
4162	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4163	pCache->TestOut.pCache));
4164	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4165	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4166	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4167	pCache->TestOut.pCtx));
4168	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4169	#endif
4170	return rc;
4171	}
4172
4173
4174	/**
4175	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4176	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4177	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4178	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4179	*
4180	* @returns VBox status code.
4181	* @param pVM Pointer to the VM.
4182	* @param pVCpu Pointer to the VMCPU.
4183	*/
4184	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4185	{
4186	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4187	{ \
4188	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4189	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4190	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4191	++cReadFields; \
4192	}
4193
4194	AssertPtr(pVM);
4195	AssertPtr(pVCpu);
4196	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4197	uint32_t cReadFields = 0;
4198
4199	/* Guest-natural selector base fields */
4200	#if 0
4201	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4202	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4204	#endif
4205	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4208	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4209	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4210	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4218	#if 0
4219	/* Unused natural width guest-state fields. */
4220	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4221	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4222	#endif
4223	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4224	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4225
4226	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4227	#if 0
4228	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4237	#endif
4238
4239	/* Natural width guest-state fields. */
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4241	#if 0
4242	/* Currently unused field. */
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4244	#endif
4245
4246	if (pVM->hm.s.fNestedPaging)
4247	{
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4249	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4250	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4251	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4252	}
4253	else
4254	{
4255	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4256	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4257	}
4258
4259	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4260	return VINF_SUCCESS;
4261	}
4262
4263
4264	/**
4265	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4266	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4267	* darwin, running 64-bit guests).
4268	*
4269	* @returns VBox status code.
4270	* @param pVCpu Pointer to the VMCPU.
4271	* @param idxField The VMCS field encoding.
4272	* @param u64Val 16, 32 or 64 bits value.
4273	*/
4274	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4275	{
4276	int rc;
4277	switch (idxField)
4278	{
4279	/*
4280	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4281	*/
4282	/* 64-bit Control fields. */
4283	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4284	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4285	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4286	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4287	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4288	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4289	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4290	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4291	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4292	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4293	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4294	case VMX_VMCS64_CTRL_EPTP_FULL:
4295	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4296	/* 64-bit Guest-state fields. */
4297	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4298	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4299	case VMX_VMCS64_GUEST_PAT_FULL:
4300	case VMX_VMCS64_GUEST_EFER_FULL:
4301	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4302	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4303	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4304	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4305	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4306	/* 64-bit Host-state fields. */
4307	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4308	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4309	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4310	{
4311	rc = VMXWriteVmcs32(idxField, u64Val);
4312	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4313	break;
4314	}
4315
4316	/*
4317	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4318	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4319	*/
4320	/* Natural-width Guest-state fields. */
4321	case VMX_VMCS_GUEST_CR3:
4322	case VMX_VMCS_GUEST_ES_BASE:
4323	case VMX_VMCS_GUEST_CS_BASE:
4324	case VMX_VMCS_GUEST_SS_BASE:
4325	case VMX_VMCS_GUEST_DS_BASE:
4326	case VMX_VMCS_GUEST_FS_BASE:
4327	case VMX_VMCS_GUEST_GS_BASE:
4328	case VMX_VMCS_GUEST_LDTR_BASE:
4329	case VMX_VMCS_GUEST_TR_BASE:
4330	case VMX_VMCS_GUEST_GDTR_BASE:
4331	case VMX_VMCS_GUEST_IDTR_BASE:
4332	case VMX_VMCS_GUEST_DR7:
4333	case VMX_VMCS_GUEST_RSP:
4334	case VMX_VMCS_GUEST_RIP:
4335	case VMX_VMCS_GUEST_SYSENTER_ESP:
4336	case VMX_VMCS_GUEST_SYSENTER_EIP:
4337	{
4338	if (!(u64Val >> 32))
4339	{
4340	/* If this field is 64-bit, VT-x will zero out the top bits. */
4341	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4342	}
4343	else
4344	{
4345	/* Assert that only the 32->64 switcher case should ever come here. */
4346	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4347	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4348	}
4349	break;
4350	}
4351
4352	default:
4353	{
4354	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4355	rc = VERR_INVALID_PARAMETER;
4356	break;
4357	}
4358	}
4359	AssertRCReturn(rc, rc);
4360	return rc;
4361	}
4362
4363
4364	/**
4365	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4366	* hosts (except darwin) for 64-bit guests.
4367	*
4368	* @param pVCpu Pointer to the VMCPU.
4369	* @param idxField The VMCS field encoding.
4370	* @param u64Val 16, 32 or 64 bits value.
4371	*/
4372	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4373	{
4374	AssertPtr(pVCpu);
4375	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4376
4377	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4378	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4379
4380	/* Make sure there are no duplicates. */
4381	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4382	{
4383	if (pCache->Write.aField[i] == idxField)
4384	{
4385	pCache->Write.aFieldVal[i] = u64Val;
4386	return VINF_SUCCESS;
4387	}
4388	}
4389
4390	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4391	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4392	pCache->Write.cValidEntries++;
4393	return VINF_SUCCESS;
4394	}
4395
4396	/* Enable later when the assembly code uses these as callbacks. */
4397	#if 0
4398	/*
4399	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4400	*
4401	* @param pVCpu Pointer to the VMCPU.
4402	* @param pCache Pointer to the VMCS cache.
4403	*
4404	* @remarks No-long-jump zone!!!
4405	*/
4406	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4407	{
4408	AssertPtr(pCache);
4409	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4410	{
4411	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4412	AssertRC(rc);
4413	}
4414	pCache->Write.cValidEntries = 0;
4415	}
4416
4417
4418	/**
4419	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4420	*
4421	* @param pVCpu Pointer to the VMCPU.
4422	* @param pCache Pointer to the VMCS cache.
4423	*
4424	* @remarks No-long-jump zone!!!
4425	*/
4426	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4427	{
4428	AssertPtr(pCache);
4429	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4430	{
4431	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4432	AssertRC(rc);
4433	}
4434	}
4435	#endif
4436	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4437
4438
4439	/**
4440	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4441	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4442	* timer.
4443	*
4444	* @returns VBox status code.
4445	* @param pVCpu Pointer to the VMCPU.
4446	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4447	* out-of-sync. Make sure to update the required fields
4448	* before using them.
4449	* @remarks No-long-jump zone!!!
4450	*/
4451	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4452	{
4453	int rc = VERR_INTERNAL_ERROR_5;
4454	bool fOffsettedTsc = false;
4455	PVM pVM = pVCpu->CTX_SUFF(pVM);
4456	if (pVM->hm.s.vmx.fUsePreemptTimer)
4457	{
4458	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4459
4460	/* Make sure the returned values have sane upper and lower boundaries. */
4461	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4462	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4463	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4464	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4465
4466	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4467	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4468	}
4469	else
4470	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4471
4472	if (fOffsettedTsc)
4473	{
4474	uint64_t u64CurTSC = ASMReadTSC();
4475	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4476	{
4477	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4478	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4479
4480	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4481	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4482	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4483	}
4484	else
4485	{
4486	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4487	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4488	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4489	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4490	}
4491	}
4492	else
4493	{
4494	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4495	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4496	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4497	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4498	}
4499	}
4500
4501
4502	/**
4503	* Determines if an exception is a contributory exception. Contributory
4504	* exceptions are ones which can cause double-faults. Page-fault is
4505	* intentionally not included here as it's a conditional contributory exception.
4506	*
4507	* @returns true if the exception is contributory, false otherwise.
4508	* @param uVector The exception vector.
4509	*/
4510	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4511	{
4512	switch (uVector)
4513	{
4514	case X86_XCPT_GP:
4515	case X86_XCPT_SS:
4516	case X86_XCPT_NP:
4517	case X86_XCPT_TS:
4518	case X86_XCPT_DE:
4519	return true;
4520	default:
4521	break;
4522	}
4523	return false;
4524	}
4525
4526
4527	/**
4528	* Sets an event as a pending event to be injected into the guest.
4529	*
4530	* @param pVCpu Pointer to the VMCPU.
4531	* @param u32IntrInfo The VM-entry interruption-information field.
4532	* @param cbInstr The VM-entry instruction length in bytes (for software
4533	* interrupts, exceptions and privileged software
4534	* exceptions).
4535	* @param u32ErrCode The VM-entry exception error code.
4536	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4537	* page-fault.
4538	*/
4539	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4540	RTGCUINTPTR GCPtrFaultAddress)
4541	{
4542	Assert(!pVCpu->hm.s.Event.fPending);
4543	pVCpu->hm.s.Event.fPending = true;
4544	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4545	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4546	pVCpu->hm.s.Event.cbInstr = cbInstr;
4547	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4548	}
4549
4550
4551	/**
4552	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4553	*
4554	* @param pVCpu Pointer to the VMCPU.
4555	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4556	* out-of-sync. Make sure to update the required fields
4557	* before using them.
4558	*/
4559	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4560	{
4561	/* Inject the double-fault. */
4562	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4563	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4564	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4565	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4566	}
4567
4568
4569	/**
4570	* Handle a condition that occurred while delivering an event through the guest
4571	* IDT.
4572	*
4573	* @returns VBox status code (informational error codes included).
4574	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4575	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4576	* continue execution of the guest which will delivery the #DF.
4577	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4578	*
4579	* @param pVCpu Pointer to the VMCPU.
4580	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4581	* out-of-sync. Make sure to update the required fields
4582	* before using them.
4583	* @param pVmxTransient Pointer to the VMX transient structure.
4584	*
4585	* @remarks No-long-jump zone!!!
4586	*/
4587	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4588	{
4589	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4590	AssertRC(rc);
4591	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4592	{
4593	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4594	AssertRCReturn(rc, rc);
4595
4596	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4597	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4598	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4599
4600	typedef enum
4601	{
4602	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4603	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4604	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4605	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4606	} VMXREFLECTXCPT;
4607
4608	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4609	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4610	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4611	{
4612	enmReflect = VMXREFLECTXCPT_XCPT;
4613	#ifdef VBOX_STRICT
4614	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4615	&& uExitVector == X86_XCPT_PF)
4616	{
4617	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4618	}
4619	#endif
4620	if ( uExitVector == X86_XCPT_PF
4621	&& uIdtVector == X86_XCPT_PF)
4622	{
4623	pVmxTransient->fVectoringPF = true;
4624	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4625	}
4626	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4627	&& hmR0VmxIsContributoryXcpt(uExitVector)
4628	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4629	\|\| uIdtVector == X86_XCPT_PF))
4630	{
4631	enmReflect = VMXREFLECTXCPT_DF;
4632	}
4633	else if (uIdtVector == X86_XCPT_DF)
4634	enmReflect = VMXREFLECTXCPT_TF;
4635	}
4636	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4637	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4638	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4639	{
4640	/*
4641	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4642	* (whatever they are) as they reoccur when restarting the instruction.
4643	*/
4644	enmReflect = VMXREFLECTXCPT_XCPT;
4645	}
4646
4647	switch (enmReflect)
4648	{
4649	case VMXREFLECTXCPT_XCPT:
4650	{
4651	uint32_t u32ErrCode = 0;
4652	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4653	{
4654	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4655	AssertRCReturn(rc, rc);
4656	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4657	}
4658
4659	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4660	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4661	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4662	rc = VINF_SUCCESS;
4663	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4664	pVCpu->hm.s.Event.u32ErrCode));
4665	break;
4666	}
4667
4668	case VMXREFLECTXCPT_DF:
4669	{
4670	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4671	rc = VINF_VMX_DOUBLE_FAULT;
4672	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4673	uIdtVector, uExitVector));
4674	break;
4675	}
4676
4677	case VMXREFLECTXCPT_TF:
4678	{
4679	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4680	rc = VINF_EM_RESET;
4681	break;
4682	}
4683
4684	default:
4685	Assert(rc == VINF_SUCCESS);
4686	break;
4687	}
4688	}
4689	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4690	return rc;
4691	}
4692
4693
4694	/**
4695	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4696	*
4697	* @returns VBox status code.
4698	* @param pVCpu Pointer to the VMCPU.
4699	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4700	* out-of-sync. Make sure to update the required fields
4701	* before using them.
4702	*
4703	* @remarks No-long-jump zone!!!
4704	*/
4705	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4706	{
4707	int rc = VINF_SUCCESS;
4708	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4709	{
4710	uint32_t uVal = 0;
4711	uint32_t uShadow = 0;
4712	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4713	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4714	AssertRCReturn(rc, rc);
4715	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4716	CPUMSetGuestCR0(pVCpu, uVal);
4717	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4718	}
4719	return rc;
4720	}
4721
4722
4723	/**
4724	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4725	*
4726	* @returns VBox status code.
4727	* @param pVCpu Pointer to the VMCPU.
4728	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4729	* out-of-sync. Make sure to update the required fields
4730	* before using them.
4731	*
4732	* @remarks No-long-jump zone!!!
4733	*/
4734	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4735	{
4736	int rc = VINF_SUCCESS;
4737	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4738	{
4739	uint32_t uVal = 0;
4740	uint32_t uShadow = 0;
4741	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4742	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4743	AssertRCReturn(rc, rc);
4744	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4745	CPUMSetGuestCR4(pVCpu, uVal);
4746	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4747	}
4748	return rc;
4749	}
4750
4751
4752	/**
4753	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4754	*
4755	* @returns VBox status code.
4756	* @param pVCpu Pointer to the VMCPU.
4757	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4758	* out-of-sync. Make sure to update the required fields
4759	* before using them.
4760	*
4761	* @remarks No-long-jump zone!!!
4762	*/
4763	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4764	{
4765	int rc = VINF_SUCCESS;
4766	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4767	{
4768	RTGCUINTREG uVal = 0;
4769	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4770	AssertRCReturn(rc, rc);
4771	pMixedCtx->rip = uVal;
4772	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4773	}
4774	return rc;
4775	}
4776
4777
4778	/**
4779	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4780	*
4781	* @returns VBox status code.
4782	* @param pVCpu Pointer to the VMCPU.
4783	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4784	* out-of-sync. Make sure to update the required fields
4785	* before using them.
4786	*
4787	* @remarks No-long-jump zone!!!
4788	*/
4789	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4790	{
4791	int rc = VINF_SUCCESS;
4792	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4793	{
4794	RTGCUINTREG uVal = 0;
4795	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4796	AssertRCReturn(rc, rc);
4797	pMixedCtx->rsp = uVal;
4798	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4799	}
4800	return rc;
4801	}
4802
4803
4804	/**
4805	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4806	*
4807	* @returns VBox status code.
4808	* @param pVCpu Pointer to the VMCPU.
4809	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4810	* out-of-sync. Make sure to update the required fields
4811	* before using them.
4812	*
4813	* @remarks No-long-jump zone!!!
4814	*/
4815	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4816	{
4817	int rc = VINF_SUCCESS;
4818	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4819	{
4820	uint32_t uVal = 0;
4821	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4822	AssertRCReturn(rc, rc);
4823	pMixedCtx->eflags.u32 = uVal;
4824
4825	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4826	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4827	{
4828	PVM pVM = pVCpu->CTX_SUFF(pVM);
4829	Assert(pVM->hm.s.vmx.pRealModeTSS);
4830	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4831	pMixedCtx->eflags.Bits.u1VM = 0;
4832	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4833	}
4834
4835	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4836	}
4837	return rc;
4838	}
4839
4840
4841	/**
4842	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4843	* guest-CPU context.
4844	*/
4845	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4846	{
4847	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4848	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4849	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4850	return rc;
4851	}
4852
4853
4854	/**
4855	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4856	* from the guest-state area in the VMCS.
4857	*
4858	* @param pVCpu Pointer to the VMCPU.
4859	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4860	* out-of-sync. Make sure to update the required fields
4861	* before using them.
4862	*
4863	* @remarks No-long-jump zone!!!
4864	*/
4865	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4866	{
4867	uint32_t uIntrState = 0;
4868	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4869	AssertRC(rc);
4870
4871	if (!uIntrState)
4872	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4873	else
4874	{
4875	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4876	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4877	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4878	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4879	AssertRC(rc);
4880	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4881	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4882	}
4883	}
4884
4885
4886	/**
4887	* Saves the guest's activity state.
4888	*
4889	* @returns VBox status code.
4890	* @param pVCpu Pointer to the VMCPU.
4891	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4892	* out-of-sync. Make sure to update the required fields
4893	* before using them.
4894	*
4895	* @remarks No-long-jump zone!!!
4896	*/
4897	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4898	{
4899	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4900	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4901	return VINF_SUCCESS;
4902	}
4903
4904
4905	/**
4906	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4907	* the current VMCS into the guest-CPU context.
4908	*
4909	* @returns VBox status code.
4910	* @param pVCpu Pointer to the VMCPU.
4911	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4912	* out-of-sync. Make sure to update the required fields
4913	* before using them.
4914	*
4915	* @remarks No-long-jump zone!!!
4916	*/
4917	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4918	{
4919	int rc = VINF_SUCCESS;
4920	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4921	{
4922	uint32_t u32Val = 0;
4923	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4924	pMixedCtx->SysEnter.cs = u32Val;
4925	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4926	}
4927
4928	RTGCUINTREG uGCVal = 0;
4929	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4930	{
4931	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4932	pMixedCtx->SysEnter.eip = uGCVal;
4933	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4934	}
4935	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4936	{
4937	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4938	pMixedCtx->SysEnter.esp = uGCVal;
4939	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4940	}
4941	return rc;
4942	}
4943
4944
4945	/**
4946	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4947	* context.
4948	*
4949	* @returns VBox status code.
4950	* @param pVCpu Pointer to the VMCPU.
4951	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4952	* out-of-sync. Make sure to update the required fields
4953	* before using them.
4954	*
4955	* @remarks No-long-jump zone!!!
4956	*/
4957	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4958	{
4959	int rc = VINF_SUCCESS;
4960	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4961	{
4962	RTGCUINTREG uVal = 0;
4963	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4964	pMixedCtx->fs.u64Base = uVal;
4965	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4966	}
4967	return rc;
4968	}
4969
4970
4971	/**
4972	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4973	* context.
4974	*
4975	* @returns VBox status code.
4976	* @param pVCpu Pointer to the VMCPU.
4977	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4978	* out-of-sync. Make sure to update the required fields
4979	* before using them.
4980	*
4981	* @remarks No-long-jump zone!!!
4982	*/
4983	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4984	{
4985	int rc = VINF_SUCCESS;
4986	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
4987	{
4988	RTGCUINTREG uVal = 0;
4989	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4990	pMixedCtx->gs.u64Base = uVal;
4991	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
4992	}
4993	return rc;
4994	}
4995
4996
4997	/**
4998	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4999	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5000	*
5001	* @returns VBox status code.
5002	* @param pVCpu Pointer to the VMCPU.
5003	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5004	* out-of-sync. Make sure to update the required fields
5005	* before using them.
5006	*
5007	* @remarks No-long-jump zone!!!
5008	*/
5009	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5010	{
5011	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5012	return VINF_SUCCESS;
5013
5014	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5015	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5016	{
5017	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5018	pMsr += i;
5019	switch (pMsr->u32IndexMSR)
5020	{
5021	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5022	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5023	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5024	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5025	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5026	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5027	default:
5028	{
5029	AssertFailed();
5030	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5031	}
5032	}
5033	}
5034	#endif
5035
5036	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5037	return VINF_SUCCESS;
5038	}
5039
5040
5041	/**
5042	* Saves the guest control registers from the current VMCS into the guest-CPU
5043	* context.
5044	*
5045	* @returns VBox status code.
5046	* @param pVCpu Pointer to the VMCPU.
5047	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5048	* out-of-sync. Make sure to update the required fields
5049	* before using them.
5050	*
5051	* @remarks No-long-jump zone!!!
5052	*/
5053	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5054	{
5055	/* Guest CR0. Guest FPU. */
5056	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5057
5058	/* Guest CR4. */
5059	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5060	AssertRCReturn(rc, rc);
5061
5062	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5063	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5064	{
5065	PVM pVM = pVCpu->CTX_SUFF(pVM);
5066	if ( pVM->hm.s.fNestedPaging
5067	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5068	{
5069	RTGCUINTREG uVal = 0;
5070	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5071	if (pMixedCtx->cr3 != uVal)
5072	{
5073	CPUMSetGuestCR3(pVCpu, uVal);
5074	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5075	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5076	}
5077
5078	/* We require EFER to check PAE mode. */
5079	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5080
5081	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5082	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5083	{
5084	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5085	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5086	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5087	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5088	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5089	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5090	}
5091	AssertRCReturn(rc, rc);
5092	}
5093
5094	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5095	}
5096	return rc;
5097	}
5098
5099
5100	/**
5101	* Reads a guest segment register from the current VMCS into the guest-CPU
5102	* context.
5103	*
5104	* @returns VBox status code.
5105	* @param pVCpu Pointer to the VMCPU.
5106	* @param idxSel Index of the selector in the VMCS.
5107	* @param idxLimit Index of the segment limit in the VMCS.
5108	* @param idxBase Index of the segment base in the VMCS.
5109	* @param idxAccess Index of the access rights of the segment in the VMCS.
5110	* @param pSelReg Pointer to the segment selector.
5111	*
5112	* @remarks No-long-jump zone!!!
5113	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5114	* as that takes care of whether to read from the VMCS cache or not.
5115	*/
5116	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5117	PCPUMSELREG pSelReg)
5118	{
5119	uint32_t u32Val = 0;
5120	int rc = VMXReadVmcs32(idxSel, &u32Val);
5121	pSelReg->Sel = (uint16_t)u32Val;
5122	pSelReg->ValidSel = (uint16_t)u32Val;
5123	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5124
5125	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5126	pSelReg->u32Limit = u32Val;
5127
5128	RTGCUINTREG uGCVal = 0;
5129	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5130	pSelReg->u64Base = uGCVal;
5131
5132	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5133	pSelReg->Attr.u = u32Val;
5134	AssertRCReturn(rc, rc);
5135
5136	/*
5137	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5138	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5139	*/
5140	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5141	{
5142	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5143	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5144	}
5145	return rc;
5146	}
5147
5148
5149	/**
5150	* Saves the guest segment registers from the current VMCS into the guest-CPU
5151	* context.
5152	*
5153	* @returns VBox status code.
5154	* @param pVCpu Pointer to the VMCPU.
5155	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5156	* out-of-sync. Make sure to update the required fields
5157	* before using them.
5158	*
5159	* @remarks No-long-jump zone!!!
5160	*/
5161	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5162	{
5163	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5164	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5165	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5166	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5167	#else
5168	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5169	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5170	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5171	#endif
5172
5173	int rc = VINF_SUCCESS;
5174
5175	/* Guest segment registers. */
5176	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5177	{
5178	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5179	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5180	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5181	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5182	rc \|= VMXLOCAL_READ_SEG(ES, es);
5183	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5184	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5185	AssertRCReturn(rc, rc);
5186
5187	/* Restore segment attributes for real-on-v86 mode hack. */
5188	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5189	{
5190	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5191	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5192	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5193	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5194	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5195	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5196	}
5197	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5198	}
5199
5200	/* Guest LDTR. */
5201	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5202	{
5203	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5204	AssertRCReturn(rc, rc);
5205	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5206	}
5207
5208	/* Guest GDTR. */
5209	RTGCUINTREG uGCVal = 0;
5210	uint32_t u32Val = 0;
5211	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5212	{
5213	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5214	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5215	pMixedCtx->gdtr.pGdt = uGCVal;
5216	pMixedCtx->gdtr.cbGdt = u32Val;
5217	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5218	}
5219
5220	/* Guest IDTR. */
5221	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5222	{
5223	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5224	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5225	pMixedCtx->idtr.pIdt = uGCVal;
5226	pMixedCtx->idtr.cbIdt = u32Val;
5227	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5228	}
5229
5230	/* Guest TR. */
5231	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5232	{
5233	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5234
5235	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5236	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5237	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5238	AssertRCReturn(rc, rc);
5239	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5240	}
5241	return rc;
5242	}
5243
5244
5245	/**
5246	* Saves the guest debug registers from the current VMCS into the guest-CPU
5247	* context.
5248	*
5249	* @returns VBox status code.
5250	* @param pVCpu Pointer to the VMCPU.
5251	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5252	* out-of-sync. Make sure to update the required fields
5253	* before using them.
5254	*
5255	* @remarks No-long-jump zone!!!
5256	*/
5257	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5258	{
5259	int rc = VINF_SUCCESS;
5260	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5261	{
5262	RTGCUINTREG uVal;
5263	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5264	pMixedCtx->dr[7] = uVal;
5265
5266	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5267	}
5268	return rc;
5269	}
5270
5271
5272	/**
5273	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5274	*
5275	* @returns VBox status code.
5276	* @param pVCpu Pointer to the VMCPU.
5277	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5278	* out-of-sync. Make sure to update the required fields
5279	* before using them.
5280	*
5281	* @remarks No-long-jump zone!!!
5282	*/
5283	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5284	{
5285	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5286	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5287	return VINF_SUCCESS;
5288	}
5289
5290
5291	/**
5292	* Saves the entire guest state from the currently active VMCS into the
5293	* guest-CPU context. This essentially VMREADs all guest-data.
5294	*
5295	* @returns VBox status code.
5296	* @param pVCpu Pointer to the VMCPU.
5297	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5298	* out-of-sync. Make sure to update the required fields
5299	* before using them.
5300	*/
5301	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5302	{
5303	Assert(pVCpu);
5304	Assert(pMixedCtx);
5305
5306	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5307	return VINF_SUCCESS;
5308
5309	VMMRZCallRing3Disable(pVCpu);
5310	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5311	LogFunc(("\n"));
5312
5313	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5315
5316	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5318
5319	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5321
5322	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5324
5325	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5327
5328	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5330
5331	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5332	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5333
5334	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5335	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5336
5337	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5338	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5339
5340	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5341	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5342
5343	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5344	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5345
5346	VMMRZCallRing3Enable(pVCpu);
5347	return rc;
5348	}
5349
5350
5351	/**
5352	* Check per-VM and per-VCPU force flag actions that require us to go back to
5353	* ring-3 for one reason or another.
5354	*
5355	* @returns VBox status code (information status code included).
5356	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5357	* ring-3.
5358	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5359	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5360	* interrupts)
5361	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5362	* all EMTs to be in ring-3.
5363	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5364	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5365	* to the EM loop.
5366	*
5367	* @param pVM Pointer to the VM.
5368	* @param pVCpu Pointer to the VMCPU.
5369	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5370	* out-of-sync. Make sure to update the required fields
5371	* before using them.
5372	*/
5373	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5374	{
5375	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5376
5377	int rc = VERR_INTERNAL_ERROR_5;
5378	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5379	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5380	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5381	{
5382	/* We need the control registers now, make sure the guest-CPU context is updated. */
5383	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5384	AssertRCReturn(rc, rc);
5385
5386	/* Pending HM CR3 sync. */
5387	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5388	{
5389	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5390	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5391	}
5392	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5393	{
5394	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5395	AssertRC(rc);
5396	}
5397
5398	/* Pending PGM C3 sync. */
5399	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5400	{
5401	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5402	if (rc != VINF_SUCCESS)
5403	{
5404	AssertRC(rc);
5405	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5406	return rc;
5407	}
5408	}
5409
5410	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5411	/* -XXX- what was that about single stepping? */
5412	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5413	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5414	{
5415	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5416	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5417	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5418	return rc;
5419	}
5420
5421	/* Pending VM request packets, such as hardware interrupts. */
5422	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5423	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5424	{
5425	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5426	return VINF_EM_PENDING_REQUEST;
5427	}
5428
5429	/* Pending PGM pool flushes. */
5430	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5431	{
5432	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5433	return VINF_PGM_POOL_FLUSH_PENDING;
5434	}
5435
5436	/* Pending DMA requests. */
5437	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5438	{
5439	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5440	return VINF_EM_RAW_TO_R3;
5441	}
5442	}
5443
5444	/* Paranoia. */
5445	Assert(rc != VERR_EM_INTERPRETER);
5446	return VINF_SUCCESS;
5447	}
5448
5449
5450	/**
5451	* Converts any TRPM trap into a pending VMX event. This is typically used when
5452	* entering from ring-3 (not longjmp returns).
5453	*
5454	* @param pVCpu Pointer to the VMCPU.
5455	* @param pCtx Pointer to the guest-CPU context.
5456	*/
5457	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5458	{
5459	if (!TRPMHasTrap(pVCpu))
5460	{
5461	Assert(!pVCpu->hm.s.Event.fPending);
5462	return;
5463	}
5464
5465	uint8_t uVector;
5466	TRPMEVENT enmTrpmEvent;
5467	RTGCUINT uErrCode;
5468	RTGCUINTPTR GCPtrFaultAddress;
5469	uint8_t cbInstr;
5470
5471	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5472	AssertRC(rc);
5473
5474	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5475	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5476	if (enmTrpmEvent == TRPM_TRAP)
5477	{
5478	switch (uVector)
5479	{
5480	case X86_XCPT_BP:
5481	case X86_XCPT_OF:
5482	{
5483	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5484	break;
5485	}
5486
5487	case X86_XCPT_PF:
5488	case X86_XCPT_DF:
5489	case X86_XCPT_TS:
5490	case X86_XCPT_NP:
5491	case X86_XCPT_SS:
5492	case X86_XCPT_GP:
5493	case X86_XCPT_AC:
5494	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5495	/* no break! */
5496	default:
5497	{
5498	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5499	break;
5500	}
5501	}
5502	}
5503	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5504	{
5505	if (uVector != X86_XCPT_NMI)
5506	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5507	else
5508	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5509	}
5510	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5511	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5512	else
5513	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5514
5515	rc = TRPMResetTrap(pVCpu);
5516	AssertRC(rc);
5517	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5518	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5519	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5520	}
5521
5522
5523	/**
5524	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5525	* VT-x to execute any instruction.
5526	*
5527	* @param pvCpu Pointer to the VMCPU.
5528	*/
5529	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5530	{
5531	if (pVCpu->hm.s.Event.fPending)
5532	{
5533	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5534	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5535	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5536	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5537
5538	/* If a trap was already pending, we did something wrong! */
5539	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5540
5541	TRPMEVENT enmTrapType;
5542	switch (uVectorType)
5543	{
5544	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5545	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5546	enmTrapType = TRPM_HARDWARE_INT;
5547	break;
5548	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5549	enmTrapType = TRPM_SOFTWARE_INT;
5550	break;
5551	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5552	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5553	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5554	enmTrapType = TRPM_TRAP;
5555	break;
5556	default:
5557	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5558	enmTrapType = TRPM_32BIT_HACK;
5559	break;
5560	}
5561
5562	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5563	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5564	AssertRC(rc);
5565
5566	if (fErrorCodeValid)
5567	TRPMSetErrorCode(pVCpu, uErrorCode);
5568	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5569	&& uVector == X86_XCPT_PF)
5570	{
5571	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5572	}
5573	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5574	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5575	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5576	{
5577	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5578	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5579	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5580	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5581	}
5582	pVCpu->hm.s.Event.fPending = false;
5583	}
5584	}
5585
5586
5587	/**
5588	* Does the necessary state syncing before doing a longjmp to ring-3.
5589	*
5590	* @param pVM Pointer to the VM.
5591	* @param pVCpu Pointer to the VMCPU.
5592	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5593	* out-of-sync. Make sure to update the required fields
5594	* before using them.
5595	* @param rcExit The reason for exiting to ring-3. Can be
5596	* VINF_VMM_UNKNOWN_RING3_CALL.
5597	*
5598	* @remarks No-long-jmp zone!!!
5599	*/
5600	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5601	{
5602	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5603	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5604
5605	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5606	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5607	AssertRC(rc);
5608
5609	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5610	if (CPUMIsGuestFPUStateActive(pVCpu))
5611	{
5612	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5613	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5614	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5615	}
5616
5617	/* Restore debug registers if necessary and resync on next R0 reentry. */
5618	if (CPUMIsGuestDebugStateActive(pVCpu))
5619	{
5620	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5621	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5622	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5623	}
5624	else if (CPUMIsHyperDebugStateActive(pVCpu))
5625	{
5626	CPUMR0LoadHostDebugState(pVM, pVCpu);
5627	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5628	}
5629
5630	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5631	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5632	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5633	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5634	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5635	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5636	}
5637
5638
5639	/**
5640	* An action requires us to go back to ring-3. This function does the necessary
5641	* steps before we can safely return to ring-3. This is not the same as longjmps
5642	* to ring-3, this is voluntary.
5643	*
5644	* @param pVM Pointer to the VM.
5645	* @param pVCpu Pointer to the VMCPU.
5646	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5647	* out-of-sync. Make sure to update the required fields
5648	* before using them.
5649	* @param rcExit The reason for exiting to ring-3. Can be
5650	* VINF_VMM_UNKNOWN_RING3_CALL.
5651	*/
5652	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5653	{
5654	Assert(pVM);
5655	Assert(pVCpu);
5656	Assert(pMixedCtx);
5657	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5658
5659	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5660	{
5661	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5662	return;
5663	}
5664	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5665	{
5666	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5667	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5668	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5669	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5670	return;
5671	}
5672
5673	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5674	VMMRZCallRing3Disable(pVCpu);
5675	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5676
5677	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5678	hmR0VmxUpdateTRPM(pVCpu);
5679
5680	/* Sync. the guest state. */
5681	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5682	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5683
5684	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5685	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5686	\| CPUM_CHANGED_LDTR
5687	\| CPUM_CHANGED_GDTR
5688	\| CPUM_CHANGED_IDTR
5689	\| CPUM_CHANGED_TR
5690	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5691
5692	/* On our way back from ring-3 the following needs to be done. */
5693	/** @todo This can change with preemption hooks. */
5694	if (rcExit == VINF_EM_RAW_INTERRUPT)
5695	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5696	else
5697	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5698
5699	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5700	VMMRZCallRing3Enable(pVCpu);
5701	}
5702
5703
5704	/**
5705	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5706	* longjump to ring-3 and possibly get preempted.
5707	*
5708	* @param pVCpu Pointer to the VMCPU.
5709	* @param enmOperation The operation causing the ring-3 longjump.
5710	* @param pvUser The user argument (pointer to the possibly
5711	* out-of-date guest-CPU context).
5712	*
5713	* @remarks Must never be called with @a enmOperation ==
5714	* VMMCALLRING3_VM_R0_ASSERTION.
5715	*/
5716	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5717	{
5718	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5719	Assert(pVCpu);
5720	Assert(pvUser);
5721	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5722	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5723
5724	VMMRZCallRing3Disable(pVCpu);
5725	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5726	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5727	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5728	VMMRZCallRing3Enable(pVCpu);
5729	}
5730
5731
5732	/**
5733	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5734	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5735	*
5736	* @returns VBox status code.
5737	* @param pVCpu Pointer to the VMCPU.
5738	*/
5739	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5740	{
5741	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5742	{
5743	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5744	{
5745	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5746	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5747	AssertRC(rc);
5748	}
5749	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5750	}
5751
5752
5753	/**
5754	* Injects any pending events into the guest if the guest is in a state to
5755	* receive them.
5756	*
5757	* @returns VBox status code (informational status codes included).
5758	* @param pVCpu Pointer to the VMCPU.
5759	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5760	* out-of-sync. Make sure to update the required fields
5761	* before using them.
5762	*/
5763	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5764	{
5765	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5766	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5767	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5768	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5769
5770	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5771	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5772	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5773	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5774
5775	int rc = VINF_SUCCESS;
5776	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5777	{
5778	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5779	bool fInject = true;
5780	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5781	{
5782	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5783	AssertRCReturn(rc, rc);
5784	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5785	if ( fBlockInt
5786	\|\| fBlockSti
5787	\|\| fBlockMovSS)
5788	{
5789	fInject = false;
5790	}
5791	}
5792	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5793	&& ( fBlockMovSS
5794	\|\| fBlockSti))
5795	{
5796	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5797	fInject = false;
5798	}
5799
5800	if (fInject)
5801	{
5802	Log(("Injecting pending event\n"));
5803	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5804	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5805	AssertRCReturn(rc, rc);
5806	pVCpu->hm.s.Event.fPending = false;
5807	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5808	}
5809	else
5810	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5811	} /** @todo SMI. SMIs take priority over NMIs. */
5812	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5813	{
5814	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5815	if ( !fBlockMovSS
5816	&& !fBlockSti)
5817	{
5818	Log(("Injecting NMI\n"));
5819	RTGCUINTPTR uIntrInfo;
5820	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5821	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5822	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5823	0 /* GCPtrFaultAddress */, &uIntrState);
5824	AssertRCReturn(rc, rc);
5825	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5826	}
5827	else
5828	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5829	}
5830	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5831	{
5832	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5833	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5834	AssertRCReturn(rc, rc);
5835	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5836	if ( !fBlockInt
5837	&& !fBlockSti
5838	&& !fBlockMovSS)
5839	{
5840	uint8_t u8Interrupt;
5841	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5842	if (RT_SUCCESS(rc))
5843	{
5844	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5845	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5846	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5847	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5848	0 /* GCPtrFaultAddress */, &uIntrState);
5849	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5850	}
5851	else
5852	{
5853	/** @todo Does this actually happen? If not turn it into an assertion. */
5854	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5855	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5856	rc = VINF_SUCCESS;
5857	}
5858	}
5859	else
5860	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5861	}
5862
5863	/*
5864	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5865	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5866	*/
5867	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5868	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5869	int rc2 = VINF_SUCCESS;
5870	if ( fBlockSti
5871	\|\| fBlockMovSS)
5872	{
5873	if (!DBGFIsStepping(pVCpu))
5874	{
5875	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5876	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5877	{
5878	/*
5879	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5880	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5881	*/
5882	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5883	}
5884	}
5885	else
5886	{
5887	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5888	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5889	uIntrState = 0;
5890	}
5891	}
5892
5893	/*
5894	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5895	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5896	*/
5897	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5898	AssertRC(rc2);
5899
5900	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5901	return rc;
5902	}
5903
5904
5905	/**
5906	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5907	*
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	*/
5913	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5914	{
5915	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5916	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5917	}
5918
5919
5920	/**
5921	* Injects a double-fault (#DF) exception into the VM.
5922	*
5923	* @returns VBox status code (informational status code included).
5924	* @param pVCpu Pointer to the VMCPU.
5925	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5926	* out-of-sync. Make sure to update the required fields
5927	* before using them.
5928	*/
5929	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5930	{
5931	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5932	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5933	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5934	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5935	puIntrState);
5936	}
5937
5938
5939	/**
5940	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5941	*
5942	* @param pVCpu Pointer to the VMCPU.
5943	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5944	* out-of-sync. Make sure to update the required fields
5945	* before using them.
5946	*/
5947	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5948	{
5949	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5950	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5951	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5952	}
5953
5954
5955	/**
5956	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
5957	*
5958	* @param pVCpu Pointer to the VMCPU.
5959	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5960	* out-of-sync. Make sure to update the required fields
5961	* before using them.
5962	* @param cbInstr The value of RIP that is to be pushed on the guest
5963	* stack.
5964	*/
5965	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5966	{
5967	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5968	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5969	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
5970	}
5971
5972
5973	/**
5974	* Injects a general-protection (#GP) fault into the VM.
5975	*
5976	* @returns VBox status code (informational status code included).
5977	* @param pVCpu Pointer to the VMCPU.
5978	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5979	* out-of-sync. Make sure to update the required fields
5980	* before using them.
5981	* @param u32ErrorCode The error code associated with the #GP.
5982	*/
5983	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
5984	uint32_t *puIntrState)
5985	{
5986	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5987	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5988	if (fErrorCodeValid)
5989	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5990	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
5991	puIntrState);
5992	}
5993
5994
5995	/**
5996	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
5997	*
5998	* @param pVCpu Pointer to the VMCPU.
5999	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6000	* out-of-sync. Make sure to update the required fields
6001	* before using them.
6002	* @param uVector The software interrupt vector number.
6003	* @param cbInstr The value of RIP that is to be pushed on the guest
6004	* stack.
6005	*/
6006	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6007	{
6008	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6009	if ( uVector == X86_XCPT_BP
6010	\|\| uVector == X86_XCPT_OF)
6011	{
6012	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6013	}
6014	else
6015	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6016	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6017	}
6018
6019
6020	/**
6021	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6022	* stack.
6023	*
6024	* @returns VBox status code (information status code included).
6025	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6026	* @param pVM Pointer to the VM.
6027	* @param pMixedCtx Pointer to the guest-CPU context.
6028	* @param uValue The value to push to the guest stack.
6029	*/
6030	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6031	{
6032	/*
6033	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6034	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6035	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6036	*/
6037	if (pMixedCtx->sp == 1)
6038	return VINF_EM_RESET;
6039	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6040	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6041	AssertRCReturn(rc, rc);
6042	return rc;
6043	}
6044
6045
6046	/**
6047	* Injects an event into the guest upon VM-entry by updating the relevant fields
6048	* in the VM-entry area in the VMCS.
6049	*
6050	* @returns VBox status code (informational error codes included).
6051	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6052	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6053	*
6054	* @param pVCpu Pointer to the VMCPU.
6055	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6056	* be out-of-sync. Make sure to update the required
6057	* fields before using them.
6058	* @param u64IntrInfo The VM-entry interruption-information field.
6059	* @param cbInstr The VM-entry instruction length in bytes (for
6060	* software interrupts, exceptions and privileged
6061	* software exceptions).
6062	* @param u32ErrCode The VM-entry exception error code.
6063	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6064	* @param puIntrState Pointer to the current guest interruptibility-state.
6065	* This interruptibility-state will be updated if
6066	* necessary. This cannot not be NULL.
6067	*
6068	* @remarks No-long-jump zone!!!
6069	* @remarks Requires CR0!
6070	*/
6071	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6072	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6073	{
6074	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6075	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6076	Assert(puIntrState);
6077	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6078
6079	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6080	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6081
6082	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6083	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6084	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6085
6086	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6087
6088	/* We require CR0 to check if the guest is in real-mode. */
6089	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6090	AssertRCReturn(rc, rc);
6091
6092	/*
6093	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6094	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6095	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6096	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6097	*/
6098	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6099	{
6100	PVM pVM = pVCpu->CTX_SUFF(pVM);
6101	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6102	{
6103	Assert(PDMVmmDevHeapIsEnabled(pVM));
6104	Assert(pVM->hm.s.vmx.pRealModeTSS);
6105
6106	/* Save the required guest state bits from the VMCS. */
6107	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6108	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6109	AssertRCReturn(rc, rc);
6110
6111	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6112	const size_t cbIdtEntry = 4;
6113	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6114	{
6115	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6116	if (uVector == X86_XCPT_DF)
6117	return VINF_EM_RESET;
6118	else if (uVector == X86_XCPT_GP)
6119	{
6120	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6121	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6122	}
6123
6124	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6125	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6126	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6127	}
6128
6129	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6130	uint16_t uGuestIp = pMixedCtx->ip;
6131	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6132	{
6133	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6134	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6135	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6136	}
6137	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6138	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6139
6140	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6141	uint16_t offIdtEntry = 0;
6142	RTSEL selIdtEntry = 0;
6143	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6144	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6145	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6146	AssertRCReturn(rc, rc);
6147
6148	/* Construct the stack frame for the interrupt/exception handler. */
6149	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6150	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6151	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6152	AssertRCReturn(rc, rc);
6153
6154	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6155	if (rc == VINF_SUCCESS)
6156	{
6157	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6158	pMixedCtx->rip = offIdtEntry;
6159	pMixedCtx->cs.Sel = selIdtEntry;
6160	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6161	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6162	&& uVector == X86_XCPT_PF)
6163	{
6164	pMixedCtx->cr2 = GCPtrFaultAddress;
6165	}
6166	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6167	\| HM_CHANGED_GUEST_RIP
6168	\| HM_CHANGED_GUEST_RFLAGS
6169	\| HM_CHANGED_GUEST_RSP;
6170
6171	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6172	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6173	{
6174	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6175	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6176	Log(("Clearing inhibition due to STI.\n"));
6177	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6178	}
6179	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6180	}
6181	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6182	return rc;
6183	}
6184	else
6185	{
6186	/*
6187	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6188	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6189	*/
6190	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6191	}
6192	}
6193
6194	/* Validate. */
6195	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6196	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6197	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6198
6199	/* Inject. */
6200	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6201	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6202	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6203	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6204
6205	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6206	&& uVector == X86_XCPT_PF)
6207	{
6208	pMixedCtx->cr2 = GCPtrFaultAddress;
6209	}
6210	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6211
6212	AssertRCReturn(rc, rc);
6213	return rc;
6214	}
6215
6216
6217	/**
6218	* Enters the VT-x session.
6219	*
6220	* @returns VBox status code.
6221	* @param pVM Pointer to the VM.
6222	* @param pVCpu Pointer to the VMCPU.
6223	* @param pCpu Pointer to the CPU info struct.
6224	*/
6225	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6226	{
6227	AssertPtr(pVM);
6228	AssertPtr(pVCpu);
6229	Assert(pVM->hm.s.vmx.fSupported);
6230	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6231	NOREF(pCpu);
6232
6233	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6234
6235	/* Make sure we're in VMX root mode. */
6236	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6237	if (!(u32HostCR4 & X86_CR4_VMXE))
6238	{
6239	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6240	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6241	}
6242
6243	/* Load the active VMCS as the current one. */
6244	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6245	if (RT_FAILURE(rc))
6246	return rc;
6247
6248	/** @todo this will change with preemption hooks where can can VMRESUME as long
6249	* as we're no preempted. */
6250	pVCpu->hm.s.fResumeVM = false;
6251	return VINF_SUCCESS;
6252	}
6253
6254
6255	/**
6256	* Leaves the VT-x session.
6257	*
6258	* @returns VBox status code.
6259	* @param pVM Pointer to the VM.
6260	* @param pVCpu Pointer to the VMCPU.
6261	* @param pCtx Pointer to the guest-CPU context.
6262	*/
6263	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6264	{
6265	AssertPtr(pVCpu);
6266	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6267	NOREF(pVM);
6268	NOREF(pCtx);
6269
6270	/** @todo this will change with preemption hooks where we only VMCLEAR when
6271	* we are actually going to be preempted, not all the time like we
6272	* currently do. */
6273	/*
6274	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6275	* and mark the VMCS launch-state as "clear".
6276	*/
6277	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6278	return rc;
6279	}
6280
6281
6282	/**
6283	* Saves the host state in the VMCS host-state.
6284	* Sets up the VM-exit MSR-load area.
6285	*
6286	* The CPU state will be loaded from these fields on every successful VM-exit.
6287	*
6288	* @returns VBox status code.
6289	* @param pVM Pointer to the VM.
6290	* @param pVCpu Pointer to the VMCPU.
6291	*
6292	* @remarks No-long-jump zone!!!
6293	*/
6294	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6295	{
6296	AssertPtr(pVM);
6297	AssertPtr(pVCpu);
6298	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6299
6300	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6301
6302	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6303	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6304	return VINF_SUCCESS;
6305
6306	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6307	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6308
6309	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6310	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6311
6312	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6314
6315	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6316	return rc;
6317	}
6318
6319
6320	/**
6321	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6322	* loaded from these fields on every successful VM-entry.
6323	*
6324	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6325	* Sets up the VM-entry controls.
6326	* Sets up the appropriate VMX non-root function to execute guest code based on
6327	* the guest CPU mode.
6328	*
6329	* @returns VBox status code.
6330	* @param pVM Pointer to the VM.
6331	* @param pVCpu Pointer to the VMCPU.
6332	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6333	* out-of-sync. Make sure to update the required fields
6334	* before using them.
6335	*
6336	* @remarks No-long-jump zone!!!
6337	*/
6338	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6339	{
6340	AssertPtr(pVM);
6341	AssertPtr(pVCpu);
6342	AssertPtr(pMixedCtx);
6343	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6344
6345	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6346
6347	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6348
6349	/* Determine real-on-v86 mode. */
6350	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6351	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6352	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6353	{
6354	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6355	}
6356
6357	/** @todo if the order of loading is important, inform it via comments here */
6358	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6359	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6360
6361	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6362	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6363
6364	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6365	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6366
6367	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6368	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6369
6370	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6371	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6372
6373	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6374	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6375
6376	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6377	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6378
6379	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6380	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6381
6382	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6384
6385	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6387
6388	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6389	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6390	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6391
6392	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6393	return rc;
6394	}
6395
6396
6397	/**
6398	* Does the preparations before executing guest code in VT-x.
6399	*
6400	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6401	* recompiler. We must be cautious what we do here regarding committing
6402	* guest-state information into the the VMCS assuming we assuredly execute the
6403	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6404	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6405	* that the recompiler can (and should) use them when it resumes guest
6406	* execution. Otherwise such operations must be done when we can no longer
6407	* exit to ring-3.
6408	*
6409	* @returns VBox status code (informational status codes included).
6410	* @retval VINF_SUCCESS if we can proceed with running the guest.
6411	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6412	* into the guest.
6413	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6414	*
6415	* @param pVM Pointer to the VM.
6416	* @param pVCpu Pointer to the VMCPU.
6417	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6418	* out-of-sync. Make sure to update the required fields
6419	* before using them.
6420	* @param pVmxTransient Pointer to the VMX transient structure.
6421	*
6422	* @remarks Called with preemption disabled.
6423	*/
6424	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6425	{
6426	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6427
6428	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6429	PGMRZDynMapFlushAutoSet(pVCpu);
6430	#endif
6431
6432	/* Check force flag actions that might require us to go back to ring-3. */
6433	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6434	if (rc != VINF_SUCCESS)
6435	return rc;
6436
6437	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6438	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6439	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6440	{
6441	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6442	RTGCPHYS GCPhysApicBase;
6443	GCPhysApicBase = pMixedCtx->msrApicBase;
6444	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6445
6446	/* Unalias any existing mapping. */
6447	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6448	AssertRCReturn(rc, rc);
6449
6450	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6451	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6452	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6453	AssertRCReturn(rc, rc);
6454
6455	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6456	}
6457
6458	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6459	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6460	pVmxTransient->uEFlags = ASMIntDisableFlags();
6461	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6462	{
6463	ASMSetFlags(pVmxTransient->uEFlags);
6464	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6465	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6466	return VINF_EM_RAW_INTERRUPT;
6467	}
6468	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6469	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6470	#endif
6471
6472	/*
6473	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6474	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6475	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6476	*/
6477	/** @todo r=bird: You reverse the effect of calling PDMGetInterrupt by
6478	* handing it over to TPRM like we do in REMR3StateBack using
6479	* TRPMAssertTrap and the other setters. */
6480	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6481	AssertRCReturn(rc, rc);
6482	return rc;
6483	}
6484
6485
6486	/**
6487	* Prepares to run guest code in VT-x and we've committed to doing so. This
6488	* means there is no backing out to ring-3 or anywhere else at this
6489	* point.
6490	*
6491	* @param pVM Pointer to the VM.
6492	* @param pVCpu Pointer to the VMCPU.
6493	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6494	* out-of-sync. Make sure to update the required fields
6495	* before using them.
6496	* @param pVmxTransient Pointer to the VMX transient structure.
6497	*
6498	* @remarks Called with preemption disabled.
6499	* @remarks No-long-jump zone!!!
6500	*/
6501	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6502	{
6503	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6504	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6505
6506	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6507	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6508	pVmxTransient->uEFlags = ASMIntDisableFlags();
6509	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6510	#endif
6511
6512	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6513	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6514	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6515	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6516	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6517	#endif
6518	int rc = VINF_SUCCESS;
6519	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6520	{
6521	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6522	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6523	}
6524	else if (pVCpu->hm.s.fContextUseFlags)
6525	{
6526	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6527	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6528	}
6529	AssertRC(rc);
6530	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6531
6532	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6533	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6534	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6535
6536	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6537	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6538	{
6539	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6540	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6541	}
6542
6543	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6544	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6545	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6546
6547	/*
6548	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6549	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6550	*/
6551	if (pVM->hm.s.fTPRPatchingActive)
6552	{
6553	Assert(!CPUMIsGuestInLongMode(pVCpu));
6554
6555	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6556	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6557	AssertRC(rc);
6558
6559	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6560	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6561	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6562	see hmR0VmxLoadGuestApicState(). */
6563	}
6564
6565	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6566	/*
6567	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6568	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6569	*/
6570	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6571	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6572	{
6573	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6574	uint64_t u64HostTscAux = 0;
6575	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6576	AssertRC(rc2);
6577	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6578	}
6579	#endif
6580
6581	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6582	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6583	to start executing. */
6584	}
6585
6586
6587	/**
6588	* Performs some essential restoration of state after running guest code in
6589	* VT-x.
6590	*
6591	* @param pVM Pointer to the VM.
6592	* @param pVCpu Pointer to the VMCPU.
6593	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6594	* out-of-sync. Make sure to update the required fields
6595	* before using them.
6596	* @param pVmxTransient Pointer to the VMX transient structure.
6597	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6598	*
6599	* @remarks Called with interrupts disabled.
6600	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6601	* unconditionally when it is safe to do so.
6602	*/
6603	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6604	{
6605	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6606	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6607
6608	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6609	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6610	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6611	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6612	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6613
6614	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6615	{
6616	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6617	/* Restore host's TSC_AUX. */
6618	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6619	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6620	#endif
6621	/** @todo Find a way to fix hardcoding a guestimate. */
6622	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6623	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6624	}
6625
6626	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6627	Assert(!(ASMGetFlags() & X86_EFL_IF));
6628	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6629
6630	/* Restore the effects of TPR patching if any. */
6631	if (pVM->hm.s.fTPRPatchingActive)
6632	{
6633	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6634	AssertRC(rc);
6635	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6636	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6637	}
6638
6639	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6640	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6641
6642	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6643	uint32_t uExitReason;
6644	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6645	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6646	AssertRC(rc);
6647	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6648	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6649
6650	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6651	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6652
6653	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6654	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6655	{
6656	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6657	return;
6658	}
6659
6660	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6661	{
6662	/* Update the guest interruptibility-state from the VMCS. */
6663	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6664	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6665	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6666	AssertRC(rc);
6667	#endif
6668	/*
6669	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6670	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6671	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6672	*/
6673	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6674	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6675	{
6676	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6677	AssertRC(rc);
6678	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6679	}
6680	}
6681	}
6682
6683
6684	/**
6685	* Runs the guest code using VT-x.
6686	*
6687	* @returns VBox status code.
6688	* @param pVM Pointer to the VM.
6689	* @param pVCpu Pointer to the VMCPU.
6690	* @param pCtx Pointer to the guest-CPU context.
6691	*
6692	* @remarks Called with preemption disabled.
6693	*/
6694	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6695	{
6696	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6697	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6698
6699	VMXTRANSIENT VmxTransient;
6700	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6701	int rc = VERR_INTERNAL_ERROR_5;
6702	uint32_t cLoops = 0;
6703	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6704
6705	for (;; cLoops++)
6706	{
6707	Assert(!HMR0SuspendPending());
6708	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6709	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6710	(unsigned)RTMpCpuId(), cLoops));
6711
6712	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6713	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6714	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6715	if (rc != VINF_SUCCESS)
6716	break;
6717
6718	/*
6719	* No longjmps to ring-3 from this point on!!!
6720	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6721	* This also disables flushing of the R0-logger instance (if any).
6722	*/
6723	VMMRZCallRing3Disable(pVCpu);
6724	VMMRZCallRing3RemoveNotification(pVCpu);
6725	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6726
6727	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6728	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6729
6730	/*
6731	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6732	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6733	*/
6734	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6735	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6736	{
6737	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6738	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6739	return rc;
6740	}
6741
6742	/* Handle the VM-exit. */
6743	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6744	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6745	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6746	#ifdef HM_PROFILE_EXIT_DISPATCH
6747	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6748	#endif
6749	#ifdef HMVMX_USE_FUNCTION_TABLE
6750	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6751	#else
6752	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6753	#endif
6754	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6755	if (rc != VINF_SUCCESS)
6756	break;
6757	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6758	{
6759	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6760	rc = VINF_EM_RAW_INTERRUPT;
6761	break;
6762	}
6763	}
6764
6765	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6766	if (rc == VERR_EM_INTERPRETER)
6767	rc = VINF_EM_RAW_EMULATE_INSTR;
6768	else if (rc == VINF_EM_RESET)
6769	rc = VINF_EM_TRIPLE_FAULT;
6770	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6771	return rc;
6772	}
6773
6774
6775	#ifndef HMVMX_USE_FUNCTION_TABLE
6776	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6777	{
6778	int rc;
6779	switch (rcReason)
6780	{
6781	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6782	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6783	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6784	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6785	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6786	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6787	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6788	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6789	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6790	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6791	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6792	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6793	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6794	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6795	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6796	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6797	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6798	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6799	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6800	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6801	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6802	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6803	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6804	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6805	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6806	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6807	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6808	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6809	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6811	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6812	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6813	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6814
6815	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6820	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6824
6825	case VMX_EXIT_VMCALL:
6826	case VMX_EXIT_VMCLEAR:
6827	case VMX_EXIT_VMLAUNCH:
6828	case VMX_EXIT_VMPTRLD:
6829	case VMX_EXIT_VMPTRST:
6830	case VMX_EXIT_VMREAD:
6831	case VMX_EXIT_VMRESUME:
6832	case VMX_EXIT_VMWRITE:
6833	case VMX_EXIT_VMXOFF:
6834	case VMX_EXIT_VMXON:
6835	case VMX_EXIT_INVEPT:
6836	case VMX_EXIT_INVVPID:
6837	case VMX_EXIT_VMFUNC:
6838	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6839	break;
6840	default:
6841	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6842	break;
6843	}
6844	return rc;
6845	}
6846	#endif
6847
6848
6849	/** Profiling macro. */
6850	#ifdef HM_PROFILE_EXIT_DISPATCH
6851	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6852	#else
6853	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6854	#endif
6855
6856
6857	#ifdef DEBUG
6858	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6859	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6860	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6861	# define VMX_ASSERT_PREEMPT_CPUID() \
6862	do \
6863	{ \
6864	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6865	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6866	} while (0)
6867
6868	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6869	do { \
6870	AssertPtr(pVCpu); \
6871	AssertPtr(pMixedCtx); \
6872	AssertPtr(pVmxTransient); \
6873	Assert(pVmxTransient->fVMEntryFailed == false); \
6874	Assert(ASMIntAreEnabled()); \
6875	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6876	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6877	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6878	(unsigned)pVCpu->idCpu)); \
6879	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6880	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6881	VMX_ASSERT_PREEMPT_CPUID(); \
6882	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6883	} while (0)
6884	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6885	do { \
6886	LogFunc(("\n")); \
6887	} while(0)
6888	#else /* Release builds */
6889	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6890	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6891	#endif
6892
6893
6894	/**
6895	* Advances the guest RIP after reading it from the VMCS.
6896	*
6897	* @returns VBox status code.
6898	* @param pVCpu Pointer to the VMCPU.
6899	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6900	* out-of-sync. Make sure to update the required fields
6901	* before using them.
6902	* @param pVmxTransient Pointer to the VMX transient structure.
6903	*
6904	* @remarks No-long-jump zone!!!
6905	*/
6906	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6907	{
6908	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6909	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6910	AssertRCReturn(rc, rc);
6911
6912	pMixedCtx->rip += pVmxTransient->cbInstr;
6913	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6914	return rc;
6915	}
6916
6917
6918	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6919	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6920	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6921	/**
6922	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6923	*/
6924	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6925	{
6926	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6928	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6929	Assert(ASMIntAreEnabled());
6930	return VINF_SUCCESS;
6931	#else
6932	return VINF_EM_RAW_INTERRUPT;
6933	#endif
6934	}
6935
6936
6937	/**
6938	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6939	*/
6940	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6941	{
6942	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6943	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6944
6945	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6946	AssertRCReturn(rc, rc);
6947
6948	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6949	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6950	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6951
6952	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6953	{
6954	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6955	return VINF_EM_RAW_INTERRUPT;
6956	}
6957
6958	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6959	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6960	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6961	{
6962	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6963	return VINF_SUCCESS;
6964	}
6965	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6966	{
6967	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6968	return rc;
6969	}
6970
6971	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6972	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6973	switch (uIntrType)
6974	{
6975	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6976	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6977	/* no break */
6978	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
6979	{
6980	switch (uVector)
6981	{
6982	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6983	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6984	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6985	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6986	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6987	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
6988	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6989	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6990	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6991	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6992	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6993	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6994	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6995	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6996	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6997	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6998	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6999	#endif
7000	default:
7001	{
7002	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7003	AssertRCReturn(rc, rc);
7004
7005	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7006	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7007	{
7008	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7009	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7010	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7011	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7012	AssertRCReturn(rc, rc);
7013	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7014	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7015	0 /* GCPtrFaultAddress */);
7016	AssertRCReturn(rc, rc);
7017	}
7018	else
7019	{
7020	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7021	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7022	}
7023	break;
7024	}
7025	}
7026	break;
7027	}
7028
7029	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7030	default:
7031	{
7032	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7033	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7034	break;
7035	}
7036	}
7037	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7038	return rc;
7039	}
7040
7041
7042	/**
7043	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7044	*/
7045	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7046	{
7047	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7048
7049	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7050	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7051	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7052	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7053	AssertRCReturn(rc, rc);
7054
7055	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7056	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7057	return VINF_SUCCESS;
7058	}
7059
7060
7061	/**
7062	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7063	*/
7064	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7065	{
7066	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7067	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7068	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7069	}
7070
7071
7072	/**
7073	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7074	*/
7075	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7076	{
7077	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7078	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7079	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7080	}
7081
7082
7083	/**
7084	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7085	*/
7086	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7087	{
7088	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7090	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7091	}
7092
7093
7094	/**
7095	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7096	*/
7097	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7098	{
7099	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7100	PVM pVM = pVCpu->CTX_SUFF(pVM);
7101	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7102	if (RT_LIKELY(rc == VINF_SUCCESS))
7103	{
7104	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7105	Assert(pVmxTransient->cbInstr == 2);
7106	}
7107	else
7108	{
7109	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7110	rc = VERR_EM_INTERPRETER;
7111	}
7112	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7113	return rc;
7114	}
7115
7116
7117	/**
7118	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7119	*/
7120	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7121	{
7122	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7123	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7124	AssertRCReturn(rc, rc);
7125
7126	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7127	return VINF_EM_RAW_EMULATE_INSTR;
7128
7129	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7130	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7131	}
7132
7133
7134	/**
7135	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7136	*/
7137	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7138	{
7139	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7140	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7141	AssertRCReturn(rc, rc);
7142
7143	PVM pVM = pVCpu->CTX_SUFF(pVM);
7144	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7145	if (RT_LIKELY(rc == VINF_SUCCESS))
7146	{
7147	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7148	Assert(pVmxTransient->cbInstr == 2);
7149	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7150	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7151	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7152	}
7153	else
7154	{
7155	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7156	rc = VERR_EM_INTERPRETER;
7157	}
7158	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7159	return rc;
7160	}
7161
7162
7163	/**
7164	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7165	*/
7166	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7167	{
7168	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7169	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7170	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7171	AssertRCReturn(rc, rc);
7172
7173	PVM pVM = pVCpu->CTX_SUFF(pVM);
7174	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7175	if (RT_LIKELY(rc == VINF_SUCCESS))
7176	{
7177	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7178	Assert(pVmxTransient->cbInstr == 3);
7179	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7180	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7181	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7182	}
7183	else
7184	{
7185	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7186	rc = VERR_EM_INTERPRETER;
7187	}
7188	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7189	return rc;
7190	}
7191
7192
7193	/**
7194	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7195	*/
7196	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7197	{
7198	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7199	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7200	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7201	AssertRCReturn(rc, rc);
7202
7203	PVM pVM = pVCpu->CTX_SUFF(pVM);
7204	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7205	if (RT_LIKELY(rc == VINF_SUCCESS))
7206	{
7207	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7208	Assert(pVmxTransient->cbInstr == 2);
7209	}
7210	else
7211	{
7212	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7213	rc = VERR_EM_INTERPRETER;
7214	}
7215	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7216	return rc;
7217	}
7218
7219
7220	/**
7221	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7222	*/
7223	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7224	{
7225	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7226	PVM pVM = pVCpu->CTX_SUFF(pVM);
7227	Assert(!pVM->hm.s.fNestedPaging);
7228
7229	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7230	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7231	AssertRCReturn(rc, rc);
7232
7233	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7234	rc = VBOXSTRICTRC_VAL(rc2);
7235	if (RT_LIKELY(rc == VINF_SUCCESS))
7236	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7237	else
7238	{
7239	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7240	pVmxTransient->uExitQualification, rc));
7241	}
7242	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7243	return rc;
7244	}
7245
7246
7247	/**
7248	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7249	*/
7250	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7251	{
7252	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7253	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7254	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7255	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7256	AssertRCReturn(rc, rc);
7257
7258	PVM pVM = pVCpu->CTX_SUFF(pVM);
7259	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7260	if (RT_LIKELY(rc == VINF_SUCCESS))
7261	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7262	else
7263	{
7264	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7265	rc = VERR_EM_INTERPRETER;
7266	}
7267	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7268	return rc;
7269	}
7270
7271
7272	/**
7273	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7274	*/
7275	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7276	{
7277	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7278	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7279	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7280	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7281	AssertRCReturn(rc, rc);
7282
7283	PVM pVM = pVCpu->CTX_SUFF(pVM);
7284	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7285	rc = VBOXSTRICTRC_VAL(rc2);
7286	if (RT_LIKELY( rc == VINF_SUCCESS
7287	\|\| rc == VINF_EM_HALT))
7288	{
7289	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7290	AssertRCReturn(rc3, rc3);
7291
7292	if ( rc == VINF_EM_HALT
7293	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7294	{
7295	rc = VINF_SUCCESS;
7296	}
7297	}
7298	else
7299	{
7300	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7301	rc = VERR_EM_INTERPRETER;
7302	}
7303	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7304	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7305	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7306	return rc;
7307	}
7308
7309
7310	/**
7311	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7312	*/
7313	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7314	{
7315	/*
7316	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7317	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7318	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7319	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7320	*/
7321	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7322	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7323	}
7324
7325
7326	/**
7327	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7328	*/
7329	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7330	{
7331	/*
7332	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7333	* root operation. If we get there there is something funny going on.
7334	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7335	*/
7336	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7337	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7338	}
7339
7340
7341	/**
7342	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7343	*/
7344	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7345	{
7346	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7347	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7348	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7349	}
7350
7351
7352	/**
7353	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7354	*/
7355	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7356	{
7357	/*
7358	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7359	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7360	* See Intel spec. 25.3 "Other Causes of VM-exits".
7361	*/
7362	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7363	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7364	}
7365
7366
7367	/**
7368	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7369	* VM-exit.
7370	*/
7371	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7372	{
7373	/*
7374	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7375	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7376	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7377	*/
7378	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7379	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7380	}
7381
7382
7383	/**
7384	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7385	* VM-exit.
7386	*/
7387	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7388	{
7389	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7390	return VINF_EM_RESET;
7391	}
7392
7393
7394	/**
7395	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7396	*/
7397	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7398	{
7399	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7400	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7401	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7402	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7403	AssertRCReturn(rc, rc);
7404
7405	pMixedCtx->rip++;
7406	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7407	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7408	rc = VINF_SUCCESS;
7409	else
7410	rc = VINF_EM_HALT;
7411
7412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7413	return rc;
7414	}
7415
7416
7417	/**
7418	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7419	*/
7420	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7421	{
7422	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7423	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7424	return VINF_SUCCESS;
7425	}
7426
7427
7428	/**
7429	* VM-exit handler for expiry of the VMX preemption timer.
7430	*/
7431	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7432	{
7433	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7434
7435	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7436	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7437
7438	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7439	PVM pVM = pVCpu->CTX_SUFF(pVM);
7440	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7441	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7442	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7443	}
7444
7445
7446	/**
7447	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7448	*/
7449	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7450	{
7451	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7452	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7453	/** @todo check if XSETBV is supported by the recompiler. */
7454	return VERR_EM_INTERPRETER;
7455	}
7456
7457
7458	/**
7459	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7460	*/
7461	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7462	{
7463	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7464	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7465	/** @todo implement EMInterpretInvpcid() */
7466	return VERR_EM_INTERPRETER;
7467	}
7468
7469
7470	/**
7471	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7472	* Error VM-exit.
7473	*/
7474	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7475	{
7476	uint32_t uIntrState;
7477	HMVMXHCUINTREG uHCReg;
7478	uint64_t u64Val;
7479	uint32_t u32Val;
7480
7481	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7482	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7483	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7484	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7485	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7486	AssertRCReturn(rc, rc);
7487
7488	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7489	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7490	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7491	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7492
7493	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7494	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7495	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7496	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7497	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7498	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7499	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7500	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7501	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7502	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7503	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7504	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7505
7506	PVM pVM = pVCpu->CTX_SUFF(pVM);
7507	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7508
7509	return VERR_VMX_INVALID_GUEST_STATE;
7510	}
7511
7512
7513	/**
7514	* VM-exit handler for VM-entry failure due to an MSR-load
7515	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7516	*/
7517	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7518	{
7519	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7520	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7521	}
7522
7523
7524	/**
7525	* VM-exit handler for VM-entry failure due to a machine-check event
7526	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7527	*/
7528	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7529	{
7530	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7531	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7532	}
7533
7534
7535	/**
7536	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7537	* theory.
7538	*/
7539	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7540	{
7541	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7542	return VERR_VMX_UNDEFINED_EXIT_CODE;
7543	}
7544
7545
7546	/**
7547	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7548	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7549	* Conditional VM-exit.
7550	*/
7551	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7552	{
7553	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7554	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7556	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7557	return VERR_EM_INTERPRETER;
7558	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7559	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7560	}
7561
7562
7563	/**
7564	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7565	*/
7566	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7567	{
7568	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7569	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7570	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7571	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7572	return VERR_EM_INTERPRETER;
7573	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7574	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7575	}
7576
7577
7578	/**
7579	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7580	*/
7581	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7582	{
7583	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7584	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7585	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7586	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7587	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7588	AssertRCReturn(rc, rc);
7589
7590	PVM pVM = pVCpu->CTX_SUFF(pVM);
7591	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7592	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7593	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7595
7596	if (RT_LIKELY(rc == VINF_SUCCESS))
7597	{
7598	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7599	Assert(pVmxTransient->cbInstr == 2);
7600	}
7601	return rc;
7602	}
7603
7604
7605	/**
7606	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7607	*/
7608	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7609	{
7610	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7611	PVM pVM = pVCpu->CTX_SUFF(pVM);
7612	int rc = VINF_SUCCESS;
7613
7614	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7615	if ( pVM->hm.s.fTPRPatchingActive
7616	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7617	{
7618	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7619	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7620	{
7621	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7622	AssertRC(rc);
7623	}
7624
7625	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7626	Assert(pVmxTransient->cbInstr == 2);
7627	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7628	return VINF_SUCCESS;
7629	}
7630
7631	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7632	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7633	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7634	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7635	AssertRCReturn(rc, rc);
7636	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7637
7638	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7639	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7641
7642	if (RT_LIKELY(rc == VINF_SUCCESS))
7643	{
7644	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7645
7646	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7647	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7648	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7649	{
7650	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7651	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7652	}
7653	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7654	{
7655	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7656	AssertRCReturn(rc, rc);
7657	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7658	}
7659	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7660	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7661
7662	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7663	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7664	{
7665	switch (pMixedCtx->ecx)
7666	{
7667	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7668	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7669	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7670	case MSR_K8_FS_BASE: /* no break */
7671	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7672	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7673	}
7674	}
7675	#ifdef VBOX_STRICT
7676	else
7677	{
7678	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7679	switch (pMixedCtx->ecx)
7680	{
7681	case MSR_IA32_SYSENTER_CS:
7682	case MSR_IA32_SYSENTER_EIP:
7683	case MSR_IA32_SYSENTER_ESP:
7684	case MSR_K8_FS_BASE:
7685	case MSR_K8_GS_BASE:
7686	{
7687	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7688	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7689	}
7690
7691	case MSR_K8_LSTAR:
7692	case MSR_K6_STAR:
7693	case MSR_K8_SF_MASK:
7694	case MSR_K8_TSC_AUX:
7695	case MSR_K8_KERNEL_GS_BASE:
7696	{
7697	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7698	pMixedCtx->ecx));
7699	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7700	}
7701	}
7702	}
7703	#endif /* VBOX_STRICT */
7704	}
7705	return rc;
7706	}
7707
7708
7709	/**
7710	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7711	*/
7712	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7713	{
7714	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7715	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7717	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7718	return VERR_EM_INTERPRETER;
7719	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7720	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7721	}
7722
7723
7724	/**
7725	* VM-exit handler for when the TPR value is lowered below the specified
7726	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7727	*/
7728	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7729	{
7730	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7731	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7732
7733	/*
7734	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7735	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7736	* resume guest execution.
7737	*/
7738	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7739	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7740	return VINF_SUCCESS;
7741	}
7742
7743
7744	/**
7745	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7746	* VM-exit.
7747	*
7748	* @retval VINF_SUCCESS when guest execution can continue.
7749	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7750	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7751	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7752	* recompiler.
7753	*/
7754	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7755	{
7756	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7757	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7758	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7759	AssertRCReturn(rc, rc);
7760
7761	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7762	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7763	PVM pVM = pVCpu->CTX_SUFF(pVM);
7764	switch (uAccessType)
7765	{
7766	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7767	{
7768	#if 0
7769	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7770	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7771	#else
7772	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7773	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7774	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7775	#endif
7776	AssertRCReturn(rc, rc);
7777
7778	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7779	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7780	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7781	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7782
7783	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7784	{
7785	case 0: /* CR0 */
7786	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7787	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7788	break;
7789	case 2: /* C2 **/
7790	/* Nothing to do here, CR2 it's not part of the VMCS. */
7791	break;
7792	case 3: /* CR3 */
7793	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7794	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7795	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7796	break;
7797	case 4: /* CR4 */
7798	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7799	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7800	break;
7801	case 8: /* CR8 */
7802	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7803	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7804	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7805	break;
7806	default:
7807	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7808	break;
7809	}
7810
7811	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7812	break;
7813	}
7814
7815	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7816	{
7817	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7818	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7819	AssertRCReturn(rc, rc);
7820	Assert( !pVM->hm.s.fNestedPaging
7821	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7822	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7823
7824	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7825	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7826	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7827
7828	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7829	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7830	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7831	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7832	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7833	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7834	break;
7835	}
7836
7837	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7838	{
7839	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7840	AssertRCReturn(rc, rc);
7841	rc = EMInterpretCLTS(pVM, pVCpu);
7842	AssertRCReturn(rc, rc);
7843	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7844	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7845	Log(("CRX CLTS write rc=%d\n", rc));
7846	break;
7847	}
7848
7849	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7850	{
7851	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7852	AssertRCReturn(rc, rc);
7853	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7854	if (RT_LIKELY(rc == VINF_SUCCESS))
7855	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7857	Log(("CRX LMSW write rc=%d\n", rc));
7858	break;
7859	}
7860
7861	default:
7862	{
7863	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7864	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7865	}
7866	}
7867
7868	/* Validate possible error codes. */
7869	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7870	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7871	if (RT_SUCCESS(rc))
7872	{
7873	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7874	AssertRCReturn(rc2, rc2);
7875	}
7876
7877	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7878	return rc;
7879	}
7880
7881
7882	/**
7883	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7884	* VM-exit.
7885	*/
7886	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7887	{
7888	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7889	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7890
7891	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7892	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7893	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7894	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7895	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7896	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7897	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7898	AssertRCReturn(rc, rc);
7899	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7900
7901	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7902	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7903	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7904	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7905	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7906	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7907	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7908
7909	/* I/O operation lookup arrays. */
7910	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7911	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7912
7913	const uint32_t cbSize = s_aIOSize[uIOWidth];
7914	const uint32_t cbInstr = pVmxTransient->cbInstr;
7915	PVM pVM = pVCpu->CTX_SUFF(pVM);
7916	if (fIOString)
7917	{
7918	/* INS/OUTS - I/O String instruction. */
7919	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7920	/** @todo for now manually disassemble later optimize by getting the fields from
7921	* the VMCS. */
7922	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7923	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7924	* segment prefix info. */
7925	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7926	if (RT_SUCCESS(rc))
7927	{
7928	if (fIOWrite)
7929	{
7930	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7931	(DISCPUMODE)pDis->uAddrMode, cbSize);
7932	rc = VBOXSTRICTRC_VAL(rc2);
7933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7934	}
7935	else
7936	{
7937	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7938	(DISCPUMODE)pDis->uAddrMode, cbSize);
7939	rc = VBOXSTRICTRC_VAL(rc2);
7940	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7941	}
7942	}
7943	else
7944	{
7945	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7946	rc = VINF_EM_RAW_EMULATE_INSTR;
7947	}
7948	}
7949	else
7950	{
7951	/* IN/OUT - I/O instruction. */
7952	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7953	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7954	if (fIOWrite)
7955	{
7956	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7957	rc = VBOXSTRICTRC_VAL(rc2);
7958	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7959	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7960	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7961	}
7962	else
7963	{
7964	uint32_t u32Result = 0;
7965	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7966	rc = VBOXSTRICTRC_VAL(rc2);
7967	if (IOM_SUCCESS(rc))
7968	{
7969	/* Save result of I/O IN instr. in AL/AX/EAX. */
7970	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7971	}
7972	else if (rc == VINF_IOM_R3_IOPORT_READ)
7973	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7975	}
7976	}
7977
7978	if (IOM_SUCCESS(rc))
7979	{
7980	pMixedCtx->rip += cbInstr;
7981	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7982	if (RT_LIKELY(rc == VINF_SUCCESS))
7983	{
7984	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7985	AssertRCReturn(rc, rc);
7986
7987	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7988	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7989	{
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7991	for (unsigned i = 0; i < 4; i++)
7992	{
7993	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7994	if ( ( uIOPort >= pMixedCtx->dr[i]
7995	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7996	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7997	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7998	{
7999	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8000	uint64_t uDR6 = ASMGetDR6();
8001
8002	/* Clear all breakpoint status flags and set the one we just hit. */
8003	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8004	uDR6 \|= (uint64_t)RT_BIT(i);
8005
8006	/*
8007	* Note: AMD64 Architecture Programmer's Manual 13.1:
8008	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8009	* be cleared by software after the contents have been read.
8010	*/
8011	ASMSetDR6(uDR6);
8012
8013	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8014	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8015
8016	/* Paranoia. */
8017	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8018	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8019	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8020
8021	/* Resync DR7 */
8022	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8023	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8024
8025	/* Set #DB to be injected into the VM and continue guest execution. */
8026	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8027	break;
8028	}
8029	}
8030	}
8031	}
8032	}
8033
8034	#ifdef DEBUG
8035	if (rc == VINF_IOM_R3_IOPORT_READ)
8036	Assert(!fIOWrite);
8037	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8038	Assert(fIOWrite);
8039	else
8040	{
8041	AssertMsg( RT_FAILURE(rc)
8042	\|\| rc == VINF_SUCCESS
8043	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8044	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8045	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8046	}
8047	#endif
8048
8049	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8050	return rc;
8051	}
8052
8053
8054	/**
8055	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8056	* VM-exit.
8057	*/
8058	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8059	{
8060	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8061
8062	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8063	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8064	AssertRCReturn(rc, rc);
8065	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8066	{
8067	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8068	AssertRCReturn(rc, rc);
8069	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8070	{
8071	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8072	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8073	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8074	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8075	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8076	{
8077	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8078	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8079
8080	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8081	Assert(!pVCpu->hm.s.Event.fPending);
8082	pVCpu->hm.s.Event.fPending = true;
8083	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8084	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8085	AssertRCReturn(rc, rc);
8086	if (fErrorCodeValid)
8087	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8088	else
8089	pVCpu->hm.s.Event.u32ErrCode = 0;
8090	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8091	&& uVector == X86_XCPT_PF)
8092	{
8093	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8094	}
8095	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8096	}
8097	}
8098	}
8099	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8100	* emulation. */
8101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8102	return VERR_EM_INTERPRETER;
8103	}
8104
8105
8106	/**
8107	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8108	*/
8109	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8110	{
8111	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8112	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8113	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8114	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8115	AssertRCReturn(rc, rc);
8116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8117	return VINF_EM_DBG_STOP;
8118	}
8119
8120
8121	/**
8122	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8123	*/
8124	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8125	{
8126	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8127
8128	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8129	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8130	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8131	return VINF_SUCCESS;
8132	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8133	return rc;
8134
8135	#if 0
8136	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8137	* just sync the whole thing. */
8138	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8139	#else
8140	/* Aggressive state sync. for now. */
8141	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8142	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8143	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8144	#endif
8145	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8146	AssertRCReturn(rc, rc);
8147
8148	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8149	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8150	switch (uAccessType)
8151	{
8152	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8153	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8154	{
8155	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8156	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8157	{
8158	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8159	}
8160
8161	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8162	GCPhys &= PAGE_BASE_GC_MASK;
8163	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8164	PVM pVM = pVCpu->CTX_SUFF(pVM);
8165	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8166	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8167
8168	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8169	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8170	CPUMCTX2CORE(pMixedCtx), GCPhys);
8171	rc = VBOXSTRICTRC_VAL(rc2);
8172	Log(("ApicAccess rc=%d\n", rc));
8173	if ( rc == VINF_SUCCESS
8174	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8175	\|\| rc == VERR_PAGE_NOT_PRESENT)
8176	{
8177	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8178	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8179	rc = VINF_SUCCESS;
8180	}
8181	break;
8182	}
8183
8184	default:
8185	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8186	rc = VINF_EM_RAW_EMULATE_INSTR;
8187	break;
8188	}
8189
8190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8191	return rc;
8192	}
8193
8194
8195	/**
8196	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8197	* VM-exit.
8198	*/
8199	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8200	{
8201	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8202
8203	/* We should -not- get this VM-exit if the guest is debugging. */
8204	if (CPUMIsGuestDebugStateActive(pVCpu))
8205	{
8206	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8207	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8208	}
8209
8210	int rc = VERR_INTERNAL_ERROR_5;
8211	if ( !DBGFIsStepping(pVCpu)
8212	&& !CPUMIsHyperDebugStateActive(pVCpu))
8213	{
8214	/* Don't intercept MOV DRx. */
8215	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8216	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8217	AssertRCReturn(rc, rc);
8218
8219	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8220	PVM pVM = pVCpu->CTX_SUFF(pVM);
8221	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8222	AssertRC(rc);
8223	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8224
8225	#ifdef VBOX_WITH_STATISTICS
8226	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8227	AssertRCReturn(rc, rc);
8228	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8229	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8230	else
8231	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8232	#endif
8233	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8234	return VINF_SUCCESS;
8235	}
8236
8237	/*
8238	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8239	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8240	*/
8241	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8242	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8243	AssertRCReturn(rc, rc);
8244
8245	PVM pVM = pVCpu->CTX_SUFF(pVM);
8246	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8247	{
8248	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8249	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8250	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8251	if (RT_SUCCESS(rc))
8252	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8253	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8254	}
8255	else
8256	{
8257	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8258	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8259	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8261	}
8262
8263	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8264	if (RT_SUCCESS(rc))
8265	{
8266	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8267	AssertRCReturn(rc2, rc2);
8268	}
8269	return rc;
8270	}
8271
8272
8273	/**
8274	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8275	* Conditional VM-exit.
8276	*/
8277	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8278	{
8279	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8280	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8281
8282	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8283	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8284	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8285	return VINF_SUCCESS;
8286	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8287	return rc;
8288
8289	RTGCPHYS GCPhys = 0;
8290	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8291
8292	#if 0
8293	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8294	#else
8295	/* Aggressive state sync. for now. */
8296	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8297	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8298	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8299	#endif
8300	AssertRCReturn(rc, rc);
8301
8302	/*
8303	* If we succeed, resume guest execution.
8304	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8305	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8306	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8307	* weird case. See @bugref{6043}.
8308	*/
8309	PVM pVM = pVCpu->CTX_SUFF(pVM);
8310	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8311	rc = VBOXSTRICTRC_VAL(rc2);
8312	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8313	if ( rc == VINF_SUCCESS
8314	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8315	\|\| rc == VERR_PAGE_NOT_PRESENT)
8316	{
8317	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8318	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8319	return VINF_SUCCESS;
8320	}
8321	return rc;
8322	}
8323
8324
8325	/**
8326	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8327	* VM-exit.
8328	*/
8329	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8330	{
8331	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8332	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8333
8334	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8335	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8336	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8337	return VINF_SUCCESS;
8338	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8339	return rc;
8340
8341	RTGCPHYS GCPhys = 0;
8342	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8343	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8344	#if 0
8345	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8346	#else
8347	/* Aggressive state sync. for now. */
8348	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8349	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8350	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8351	#endif
8352	AssertRCReturn(rc, rc);
8353
8354	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8355	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8356
8357	RTGCUINT uErrorCode = 0;
8358	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8359	uErrorCode \|= X86_TRAP_PF_ID;
8360	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8361	uErrorCode \|= X86_TRAP_PF_RW;
8362	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8363	uErrorCode \|= X86_TRAP_PF_P;
8364
8365	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8366
8367	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8368	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8369
8370	/* Handle the pagefault trap for the nested shadow table. */
8371	PVM pVM = pVCpu->CTX_SUFF(pVM);
8372	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8373	TRPMResetTrap(pVCpu);
8374
8375	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8376	if ( rc == VINF_SUCCESS
8377	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8378	\|\| rc == VERR_PAGE_NOT_PRESENT)
8379	{
8380	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8381	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8382	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8383	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8384	return VINF_SUCCESS;
8385	}
8386
8387	Log(("EPT return to ring-3 rc=%d\n"));
8388	return rc;
8389	}
8390
8391
8392	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8393	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8394	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8395	/**
8396	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8397	*/
8398	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8399	{
8400	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8401	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8402
8403	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8404	AssertRCReturn(rc, rc);
8405
8406	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8407	{
8408	/* Old-style FPU error reporting needs some extra work. */
8409	/** @todo don't fall back to the recompiler, but do it manually. */
8410	return VERR_EM_INTERPRETER;
8411	}
8412	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8413	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8414	return rc;
8415	}
8416
8417
8418	/**
8419	* VM-exit exception handler for #BP (Breakpoint exception).
8420	*/
8421	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8422	{
8423	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8424	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8425
8426	/** @todo Try optimize this by not saving the entire guest state unless
8427	* really needed. */
8428	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8429	AssertRCReturn(rc, rc);
8430
8431	PVM pVM = pVCpu->CTX_SUFF(pVM);
8432	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8433	if (rc == VINF_EM_RAW_GUEST_TRAP)
8434	{
8435	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8436	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8437	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8438	AssertRCReturn(rc, rc);
8439
8440	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8441	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8442	}
8443
8444	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8445	return rc;
8446	}
8447
8448
8449	/**
8450	* VM-exit exception handler for #DB (Debug exception).
8451	*/
8452	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8453	{
8454	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8456
8457	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8458	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8459	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8460	AssertRCReturn(rc, rc);
8461
8462	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8463	uint64_t uDR6 = X86_DR6_INIT_VAL;
8464	uDR6 \|= (pVmxTransient->uExitQualification
8465	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8466	PVM pVM = pVCpu->CTX_SUFF(pVM);
8467	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8468	if (rc == VINF_EM_RAW_GUEST_TRAP)
8469	{
8470	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8471	pMixedCtx->dr[6] = uDR6;
8472
8473	if (CPUMIsGuestDebugStateActive(pVCpu))
8474	ASMSetDR6(pMixedCtx->dr[6]);
8475
8476	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8477
8478	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8479	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8480
8481	/* Paranoia. */
8482	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8483	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8484	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8485
8486	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8487	AssertRCReturn(rc,rc);
8488
8489	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8490	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8491	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8492	AssertRCReturn(rc2, rc2);
8493	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8494	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8495	rc = VINF_SUCCESS;
8496	}
8497
8498	return rc;
8499	}
8500
8501
8502	/**
8503	* VM-exit exception handler for #NM (Device-not-available exception: floating
8504	* point exception).
8505	*/
8506	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8507	{
8508	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8509
8510	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8511	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8512	#endif
8513
8514	/* We require CR0 and EFER. EFER is always up-to-date. */
8515	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8516	AssertRCReturn(rc, rc);
8517
8518	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8519	PVM pVM = pVCpu->CTX_SUFF(pVM);
8520	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8521	if (rc == VINF_SUCCESS)
8522	{
8523	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8524	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8526	return VINF_SUCCESS;
8527	}
8528
8529	/* Forward #NM to the guest. */
8530	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8531	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8532	AssertRCReturn(rc, rc);
8533	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8534	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8535	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8536	return rc;
8537	}
8538
8539
8540	/**
8541	* VM-exit exception handler for #GP (General-protection exception).
8542	*
8543	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8544	*/
8545	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8546	{
8547	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8548	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8549
8550	int rc = VERR_INTERNAL_ERROR_5;
8551	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8552	{
8553	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8554	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8555	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8556	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8557	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8558	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8559	AssertRCReturn(rc, rc);
8560	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8561	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8562	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8563	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8564	return rc;
8565	#else
8566	/* We don't intercept #GP. */
8567	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8568	return VERR_VMX_UNEXPECTED_EXCEPTION;
8569	#endif
8570	}
8571
8572	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8573	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8574
8575	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8576	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8577	AssertRCReturn(rc, rc);
8578
8579	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8580	uint32_t cbOp = 0;
8581	PVM pVM = pVCpu->CTX_SUFF(pVM);
8582	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8583	if (RT_SUCCESS(rc))
8584	{
8585	rc = VINF_SUCCESS;
8586	Assert(cbOp == pDis->cbInstr);
8587	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8588	switch (pDis->pCurInstr->uOpcode)
8589	{
8590	case OP_CLI:
8591	pMixedCtx->eflags.Bits.u1IF = 0;
8592	pMixedCtx->rip += pDis->cbInstr;
8593	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8594	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8595	break;
8596
8597	case OP_STI:
8598	pMixedCtx->eflags.Bits.u1IF = 1;
8599	pMixedCtx->rip += pDis->cbInstr;
8600	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8601	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8602	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8604	break;
8605
8606	case OP_HLT:
8607	rc = VINF_EM_HALT;
8608	pMixedCtx->rip += pDis->cbInstr;
8609	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8611	break;
8612
8613	case OP_POPF:
8614	{
8615	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8616	uint32_t cbParm = 0;
8617	uint32_t uMask = 0;
8618	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8619	{
8620	cbParm = 4;
8621	uMask = 0xffffffff;
8622	}
8623	else
8624	{
8625	cbParm = 2;
8626	uMask = 0xffff;
8627	}
8628
8629	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8630	RTGCPTR GCPtrStack = 0;
8631	X86EFLAGS uEflags;
8632	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8633	&GCPtrStack);
8634	if (RT_SUCCESS(rc))
8635	{
8636	Assert(sizeof(uEflags.u32) >= cbParm);
8637	uEflags.u32 = 0;
8638	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8639	}
8640	if (RT_FAILURE(rc))
8641	{
8642	rc = VERR_EM_INTERPRETER;
8643	break;
8644	}
8645	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8646	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8647	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8648	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8649	pMixedCtx->eflags.Bits.u1RF = 0;
8650	pMixedCtx->esp += cbParm;
8651	pMixedCtx->esp &= uMask;
8652	pMixedCtx->rip += pDis->cbInstr;
8653	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8654	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8655	break;
8656	}
8657
8658	case OP_PUSHF:
8659	{
8660	uint32_t cbParm = 0;
8661	uint32_t uMask = 0;
8662	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8663	{
8664	cbParm = 4;
8665	uMask = 0xffffffff;
8666	}
8667	else
8668	{
8669	cbParm = 2;
8670	uMask = 0xffff;
8671	}
8672
8673	/* Get the stack pointer & push the contents of eflags onto the stack. */
8674	RTGCPTR GCPtrStack = 0;
8675	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8676	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8677	if (RT_FAILURE(rc))
8678	{
8679	rc = VERR_EM_INTERPRETER;
8680	break;
8681	}
8682	X86EFLAGS uEflags;
8683	uEflags = pMixedCtx->eflags;
8684	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8685	uEflags.Bits.u1RF = 0;
8686	uEflags.Bits.u1VM = 0;
8687
8688	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8689	if (RT_FAILURE(rc))
8690	{
8691	rc = VERR_EM_INTERPRETER;
8692	break;
8693	}
8694	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8695	pMixedCtx->esp -= cbParm;
8696	pMixedCtx->esp &= uMask;
8697	pMixedCtx->rip += pDis->cbInstr;
8698	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8699	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8700	break;
8701	}
8702
8703	case OP_IRET:
8704	{
8705	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8706	* instruction reference. */
8707	RTGCPTR GCPtrStack = 0;
8708	uint32_t uMask = 0xffff;
8709	uint16_t aIretFrame[3];
8710	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8711	{
8712	rc = VERR_EM_INTERPRETER;
8713	break;
8714	}
8715	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8716	&GCPtrStack);
8717	if (RT_SUCCESS(rc))
8718	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8719	if (RT_FAILURE(rc))
8720	{
8721	rc = VERR_EM_INTERPRETER;
8722	break;
8723	}
8724	pMixedCtx->eip = 0;
8725	pMixedCtx->ip = aIretFrame[0];
8726	pMixedCtx->cs.Sel = aIretFrame[1];
8727	pMixedCtx->cs.ValidSel = aIretFrame[1];
8728	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8729	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8730	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8731	pMixedCtx->sp += sizeof(aIretFrame);
8732	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8733	\| HM_CHANGED_GUEST_RFLAGS;
8734	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8735	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8736	break;
8737	}
8738
8739	case OP_INT:
8740	{
8741	uint16_t uVector = pDis->Param1.uValue & 0xff;
8742	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8743	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8744	break;
8745	}
8746
8747	case OP_INTO:
8748	{
8749	if (pMixedCtx->eflags.Bits.u1OF)
8750	{
8751	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8752	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8753	}
8754	break;
8755	}
8756
8757	default:
8758	{
8759	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8760	EMCODETYPE_SUPERVISOR);
8761	rc = VBOXSTRICTRC_VAL(rc2);
8762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8763	Log(("#GP rc=%Rrc\n", rc));
8764	break;
8765	}
8766	}
8767	}
8768	else
8769	rc = VERR_EM_INTERPRETER;
8770
8771	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8772	("#GP Unexpected rc=%Rrc\n", rc));
8773	return rc;
8774	}
8775
8776
8777	/**
8778	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8779	* the exception reported in the VMX transient structure back into the VM.
8780	*
8781	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8782	* VMX transient structure to be up-to-date.
8783	*/
8784	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8785	{
8786	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8787
8788	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8789	hmR0VmxCheckExitDueToEventDelivery(). */
8790	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8791	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8792	AssertRCReturn(rc, rc);
8793	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8794	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8795	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8796	return VINF_SUCCESS;
8797	}
8798
8799
8800	/**
8801	* VM-exit exception handler for #PF (Page-fault exception).
8802	*/
8803	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8804	{
8805	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8806	PVM pVM = pVCpu->CTX_SUFF(pVM);
8807	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8808	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8809	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8810	AssertRCReturn(rc, rc);
8811
8812	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8813	if (pVM->hm.s.fNestedPaging)
8814	{
8815	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8816	{
8817	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8818	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8819	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8820	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8821	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8822	}
8823	else
8824	{
8825	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8826	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8827	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8828	Log(("Pending #DF due to vectoring #PF. NP\n"));
8829	}
8830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8831	return rc;
8832	}
8833	#else
8834	Assert(!pVM->hm.s.fNestedPaging);
8835	#endif
8836
8837	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8838	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8839	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8840	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8841	AssertRCReturn(rc, rc);
8842	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8843	if ( pVM->hm.s.fTRPPatchingAllowed
8844	&& pVM->hm.s.pGuestPatchMem
8845	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8846	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8847	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8848	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8849	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8850	{
8851	RTGCPHYS GCPhys;
8852	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8853	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8854	if ( rc == VINF_SUCCESS
8855	&& GCPhys == GCPhysApicBase)
8856	{
8857	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8858	AssertRCReturn(rc, rc);
8859
8860	/* Only attempt to patch the instruction once. */
8861	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8862	if (!pPatch)
8863	return VINF_EM_HM_PATCH_TPR_INSTR;
8864	}
8865	}
8866	#endif
8867
8868	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8869	AssertRCReturn(rc, rc);
8870
8871	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8872	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8873
8874	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8875	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8876	(RTGCPTR)pVmxTransient->uExitQualification);
8877
8878	Log(("#PF: rc=%Rrc\n", rc));
8879	if (rc == VINF_SUCCESS)
8880	{
8881	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8882	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8883	* memory? We don't update the whole state here... */
8884	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8885	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8886	TRPMResetTrap(pVCpu);
8887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8888	return rc;
8889	}
8890	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8891	{
8892	if (!pVmxTransient->fVectoringPF)
8893	{
8894	/* It's a guest page fault and needs to be reflected to the guest. */
8895	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8896	TRPMResetTrap(pVCpu);
8897	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8898	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8899	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8900	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8901	}
8902	else
8903	{
8904	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8905	TRPMResetTrap(pVCpu);
8906	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8907	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8908	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8909	}
8910
8911	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8912	return VINF_SUCCESS;
8913	}
8914
8915	TRPMResetTrap(pVCpu);
8916	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8917	return rc;
8918	}
8919

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45857

Download in other formats: