HMVMXR0.cpp@ 45849

Last change on this file since 45849 was 45849, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Redundant.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45849 2013-04-30 18:01:53Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147
148	/*******************************************************************************
149	* Structures and Typedefs *
150	*******************************************************************************/
151	/**
152	* A state structure for holding miscellaneous information across
153	* VMX non-root operation and restored after the transition.
154	*/
155	typedef struct VMXTRANSIENT
156	{
157	/** The host's rflags/eflags. */
158	RTCCUINTREG uEFlags;
159	#if HC_ARCH_BITS == 32
160	uint32_t u32Alignment0;
161	#endif
162	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
163	uint64_t u64LStarMsr;
164	/** The guest's TPR value used for TPR shadowing. */
165	uint8_t u8GuestTpr;
166	/** Alignment. */
167	uint8_t abAlignment0[6];
168
169	/** The basic VM-exit reason. */
170	uint16_t uExitReason;
171	/** Alignment. */
172	uint16_t u16Alignment0;
173	/** The VM-exit interruption error code. */
174	uint32_t uExitIntrErrorCode;
175	/** The VM-exit exit qualification. */
176	RTGCUINTPTR uExitQualification;
177	#if GC_ARCH_BITS == 32
178	/** Alignment. */
179	uint32_t u32Alignment1;
180	#endif
181
182	/** The VM-exit interruption-information field. */
183	uint32_t uExitIntrInfo;
184	/** The VM-exit instruction-length field. */
185	uint32_t cbInstr;
186	/** Whether the VM-entry failed or not. */
187	bool fVMEntryFailed;
188	/** Alignment. */
189	uint8_t abAlignment1[5];
190
191	/** The VM-entry interruption-information field. */
192	uint32_t uEntryIntrInfo;
193	/** The VM-entry exception error code field. */
194	uint32_t uEntryXcptErrorCode;
195	/** The VM-entry instruction length field. */
196	uint32_t cbEntryInstr;
197
198	/** IDT-vectoring information field. */
199	uint32_t uIdtVectoringInfo;
200	/** IDT-vectoring error code. */
201	uint32_t uIdtVectoringErrorCode;
202
203	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
204	uint32_t fVmcsFieldsRead;
205	/** Whether TSC-offsetting should be setup before VM-entry. */
206	bool fUpdateTscOffsettingAndPreemptTimer;
207	/** Whether the VM-exit was caused by a page-fault during delivery of a
208	* contributary exception or a page-fault. */
209	bool fVectoringPF;
210	} VMXTRANSIENT, *PVMXTRANSIENT;
211	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
212	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
213	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
214
215
216	/**
217	* MSR-bitmap read permissions.
218	*/
219	typedef enum VMXMSREXITREAD
220	{
221	/** Reading this MSR causes a VM-exit. */
222	VMXMSREXIT_INTERCEPT_READ = 0xb,
223	/** Reading this MSR does not cause a VM-exit. */
224	VMXMSREXIT_PASSTHRU_READ
225	} VMXMSREXITREAD;
226
227	/**
228	* MSR-bitmap write permissions.
229	*/
230	typedef enum VMXMSREXITWRITE
231	{
232	/** Writing to this MSR causes a VM-exit. */
233	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
234	/** Writing to this MSR does not cause a VM-exit. */
235	VMXMSREXIT_PASSTHRU_WRITE
236	} VMXMSREXITWRITE;
237
238
239	/*******************************************************************************
240	* Internal Functions *
241	*******************************************************************************/
242	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
243	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
244	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
245	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
246	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
247	#endif
248	#ifndef HMVMX_USE_FUNCTION_TABLE
249	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
250	#endif
251
252	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296
297	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304
305
306	/*******************************************************************************
307	* Global Variables *
308	*******************************************************************************/
309	#ifdef HMVMX_USE_FUNCTION_TABLE
310	/**
311	* VM-exit handler.
312	*
313	* @returns VBox status code.
314	* @param pVCpu Pointer to the VMCPU.
315	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
316	* out-of-sync. Make sure to update the required
317	* fields before using them.
318	* @param pVmxTransient Pointer to the VMX-transient structure.
319	*/
320	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	/** Pointer to VM-exit handler. */
322	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
323
324	/**
325	* VMX_EXIT dispatch table.
326	*/
327	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
328	{
329	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
330	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
331	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
332	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
333	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
334	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
335	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
336	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
337	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
338	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
339	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
340	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
341	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
342	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
343	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
344	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
345	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
346	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
347	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
348	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
349	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
350	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
351	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
352	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
353	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
354	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
355	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
356	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
357	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
358	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
359	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
360	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
361	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
362	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
363	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
364	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
365	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
366	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
367	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
368	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
369	/* 40 UNDEFINED */ hmR0VmxExitPause,
370	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
371	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
372	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
373	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
374	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
376	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
377	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
378	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
379	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
380	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
381	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
382	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
383	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
384	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
385	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
387	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
388	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
389	};
390	#endif /* HMVMX_USE_FUNCTION_TABLE */
391
392	#ifdef VBOX_STRICT
393	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
394	{
395	/* 0 */ "(Not Used)",
396	/* 1 */ "VMCALL executed in VMX root operation.",
397	/* 2 */ "VMCLEAR with invalid physical address.",
398	/* 3 */ "VMCLEAR with VMXON pointer.",
399	/* 4 */ "VMLAUNCH with non-clear VMCS.",
400	/* 5 */ "VMRESUME with non-launched VMCS.",
401	/* 6 */ "VMRESUME after VMXOFF",
402	/* 7 */ "VM entry with invalid control fields.",
403	/* 8 */ "VM entry with invalid host state fields.",
404	/* 9 */ "VMPTRLD with invalid physical address.",
405	/* 10 */ "VMPTRLD with VMXON pointer.",
406	/* 11 */ "VMPTRLD with incorrect revision identifier.",
407	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
408	/* 13 */ "VMWRITE to read-only VMCS component.",
409	/* 14 */ "(Not Used)",
410	/* 15 */ "VMXON executed in VMX root operation.",
411	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
412	/* 17 */ "VM entry with non-launched executing VMCS.",
413	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
414	/* 19 */ "VMCALL with non-clear VMCS.",
415	/* 20 */ "VMCALL with invalid VM-exit control fields.",
416	/* 21 */ "(Not Used)",
417	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
418	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
419	/* 24 */ "VMCALL with invalid SMM-monitor features.",
420	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
421	/* 26 */ "VM entry with events blocked by MOV SS.",
422	/* 27 */ "(Not Used)",
423	/* 28 */ "Invalid operand to INVEPT/INVVPID."
424	};
425	#endif /* VBOX_STRICT */
426
427
428
429	/**
430	* Updates the VM's last error record. If there was a VMX instruction error,
431	* reads the error data from the VMCS and updates VCPU's last error record as
432	* well.
433	*
434	* @param pVM Pointer to the VM.
435	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
436	* VERR_VMX_UNABLE_TO_START_VM or
437	* VERR_VMX_INVALID_VMCS_FIELD).
438	* @param rc The error code.
439	*/
440	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
441	{
442	AssertPtr(pVM);
443	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
444	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
445	{
446	AssertPtrReturnVoid(pVCpu);
447	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
448	}
449	pVM->hm.s.lLastError = rc;
450	}
451
452
453	/**
454	* Reads the VM-entry interruption-information field from the VMCS into the VMX
455	* transient structure.
456	*
457	* @returns VBox status code.
458	* @param pVmxTransient Pointer to the VMX transient structure.
459	*
460	* @remarks No-long-jump zone!!!
461	*/
462	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
463	{
464	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
465	AssertRCReturn(rc, rc);
466	return VINF_SUCCESS;
467	}
468
469
470	/**
471	* Reads the VM-entry exception error code field from the VMCS into
472	* the VMX transient structure.
473	*
474	* @returns VBox status code.
475	* @param pVmxTransient Pointer to the VMX transient structure.
476	*
477	* @remarks No-long-jump zone!!!
478	*/
479	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
480	{
481	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
482	AssertRCReturn(rc, rc);
483	return VINF_SUCCESS;
484	}
485
486
487	/**
488	* Reads the VM-entry exception error code field from the VMCS into
489	* the VMX transient structure.
490	*
491	* @returns VBox status code.
492	* @param pVCpu Pointer to the VMCPU.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*
495	* @remarks No-long-jump zone!!!
496	*/
497	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
500	AssertRCReturn(rc, rc);
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption-information field from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit interruption error code from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the VM-exit instruction length field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVCpu Pointer to the VMCPU.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*/
553	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
554	{
555	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
558	AssertRCReturn(rc, rc);
559	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
560	}
561	return VINF_SUCCESS;
562	}
563
564
565	/**
566	* Reads the exit qualification from the VMCS into the VMX transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVCpu Pointer to the VMCPU.
570	* @param pVmxTransient Pointer to the VMX transient structure.
571	*/
572	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
573	{
574	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
575	{
576	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
577	AssertRCReturn(rc, rc);
578	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
579	}
580	return VINF_SUCCESS;
581	}
582
583
584	/**
585	* Reads the IDT-vectoring information field from the VMCS into the VMX
586	* transient structure.
587	*
588	* @returns VBox status code.
589	* @param pVmxTransient Pointer to the VMX transient structure.
590	*
591	* @remarks No-long-jump zone!!!
592	*/
593	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
594	{
595	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
596	{
597	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
598	AssertRCReturn(rc, rc);
599	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
600	}
601	return VINF_SUCCESS;
602	}
603
604
605	/**
606	* Reads the IDT-vectoring error code from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*/
612	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
613	{
614	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
617	AssertRCReturn(rc, rc);
618	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
619	}
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Enters VMX root mode operation on the current CPU.
626	*
627	* @returns VBox status code.
628	* @param pVM Pointer to the VM (optional, can be NULL, after
629	* a resume).
630	* @param HCPhysCpuPage Physical address of the VMXON region.
631	* @param pvCpuPage Pointer to the VMXON region.
632	*/
633	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
634	{
635	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
636	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
637	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
638
639	if (pVM)
640	{
641	/* Write the VMCS revision dword to the VMXON region. */
642	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
643	}
644
645	/* Enable the VMX bit in CR4 if necessary. */
646	RTCCUINTREG uCr4 = ASMGetCR4();
647	if (!(uCr4 & X86_CR4_VMXE))
648	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
649
650	/* Enter VMX root mode. */
651	int rc = VMXEnable(HCPhysCpuPage);
652	if (RT_FAILURE(rc))
653	ASMSetCR4(uCr4);
654
655	return rc;
656	}
657
658
659	/**
660	* Exits VMX root mode operation on the current CPU.
661	*
662	* @returns VBox status code.
663	*/
664	static int hmR0VmxLeaveRootMode(void)
665	{
666	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
667
668	/* If we're for some reason not in VMX root mode, then don't leave it. */
669	if (ASMGetCR4() & X86_CR4_VMXE)
670	{
671	/* Exit VMX root mode and clear the VMX bit in CR4 */
672	VMXDisable();
673	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
674	return VINF_SUCCESS;
675	}
676
677	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
678	}
679
680
681	/**
682	* Allocates and maps one physically contiguous page. The allocated page is
683	* zero'd out. (Used by various VT-x structures).
684	*
685	* @returns IPRT status code.
686	* @param pMemObj Pointer to the ring-0 memory object.
687	* @param ppVirt Where to store the virtual address of the
688	* allocation.
689	* @param pPhys Where to store the physical address of the
690	* allocation.
691	*/
692	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
693	{
694	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
695	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
696	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
697
698	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
699	if (RT_FAILURE(rc))
700	return rc;
701	ppVirt = RTR0MemObjAddress(pMemObj);
702	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
703	ASMMemZero32(*ppVirt, PAGE_SIZE);
704	return VINF_SUCCESS;
705	}
706
707
708	/**
709	* Frees and unmaps an allocated physical page.
710	*
711	* @param pMemObj Pointer to the ring-0 memory object.
712	* @param ppVirt Where to re-initialize the virtual address of
713	* allocation as 0.
714	* @param pHCPhys Where to re-initialize the physical address of the
715	* allocation as 0.
716	*/
717	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
718	{
719	AssertPtr(pMemObj);
720	AssertPtr(ppVirt);
721	AssertPtr(pHCPhys);
722	if (*pMemObj != NIL_RTR0MEMOBJ)
723	{
724	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
725	AssertRC(rc);
726	*pMemObj = NIL_RTR0MEMOBJ;
727	*ppVirt = 0;
728	*pHCPhys = 0;
729	}
730	}
731
732
733	/**
734	* Worker function to free VT-x related structures.
735	*
736	* @returns IPRT status code.
737	* @param pVM Pointer to the VM.
738	*/
739	static void hmR0VmxStructsFree(PVM pVM)
740	{
741	for (VMCPUID i = 0; i < pVM->cCpus; i++)
742	{
743	PVMCPU pVCpu = &pVM->aCpus[i];
744	AssertPtr(pVCpu);
745
746	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
747	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
748	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
749	#endif
750
751	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
752	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
753
754	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
755	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
756	}
757
758	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
759	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
760	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
761	#endif
762	}
763
764
765	/**
766	* Worker function to allocate VT-x related VM structures.
767	*
768	* @returns IPRT status code.
769	* @param pVM Pointer to the VM.
770	*/
771	static int hmR0VmxStructsAlloc(PVM pVM)
772	{
773	/*
774	* Initialize members up-front so we can cleanup properly on allocation failure.
775	*/
776	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
777	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
778	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
779	pVM->hm.s.vmx.HCPhys##a_Name = 0;
780
781	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
782	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
783	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
784	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
785
786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
787	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
788	#endif
789	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
790
791	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
792	for (VMCPUID i = 0; i < pVM->cCpus; i++)
793	{
794	PVMCPU pVCpu = &pVM->aCpus[i];
795	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
796	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
797	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
798	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
799	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
800	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
801	#endif
802	}
803	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
804	#undef VMXLOCAL_INIT_VM_MEMOBJ
805
806	/*
807	* Allocate all the VT-x structures.
808	*/
809	int rc = VINF_SUCCESS;
810	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
811	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
812	if (RT_FAILURE(rc))
813	goto cleanup;
814	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
815	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
816	#endif
817
818	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
820	{
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
822	&pVM->hm.s.vmx.HCPhysApicAccess);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	}
826
827	/*
828	* Initialize per-VCPU VT-x structures.
829	*/
830	for (VMCPUID i =0; i < pVM->cCpus; i++)
831	{
832	PVMCPU pVCpu = &pVM->aCpus[i];
833	AssertPtr(pVCpu);
834
835	/* Allocate the VM control structure (VMCS). */
836	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
837	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840
841	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
842	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
843	{
844	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
845	&pVCpu->hm.s.vmx.HCPhysVirtApic);
846	if (RT_FAILURE(rc))
847	goto cleanup;
848	}
849
850	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
852	{
853	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
854	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
855	if (RT_FAILURE(rc))
856	goto cleanup;
857	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
858	}
859
860	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
861	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
862	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
863	if (RT_FAILURE(rc))
864	goto cleanup;
865
866	/* Allocate the VM-exit MSR-load page for the host MSRs. */
867	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
868	if (RT_FAILURE(rc))
869	goto cleanup;
870	#endif
871	}
872
873	return VINF_SUCCESS;
874
875	cleanup:
876	hmR0VmxStructsFree(pVM);
877	return rc;
878	}
879
880
881	/**
882	* Does global VT-x initialization (called during module initialization).
883	*
884	* @returns VBox status code.
885	*/
886	VMMR0DECL(int) VMXR0GlobalInit(void)
887	{
888	#ifdef HMVMX_USE_FUNCTION_TABLE
889	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
890	# ifdef VBOX_STRICT
891	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
892	Assert(g_apfnVMExitHandlers[i]);
893	# endif
894	#endif
895	return VINF_SUCCESS;
896	}
897
898
899	/**
900	* Does global VT-x termination (called during module termination).
901	*/
902	VMMR0DECL(void) VMXR0GlobalTerm()
903	{
904	/* Nothing to do currently. */
905	}
906
907
908	/**
909	* Sets up and activates VT-x on the current CPU.
910	*
911	* @returns VBox status code.
912	* @param pCpu Pointer to the global CPU info struct.
913	* @param pVM Pointer to the VM (can be NULL after a host resume
914	* operation).
915	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
916	* fEnabledByHost is true).
917	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
918	* @a fEnabledByHost is true).
919	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
920	* enable VT-x/AMD-V on the host.
921	*/
922	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
923	{
924	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
925	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
926
927	if (!fEnabledByHost)
928	{
929	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
930	if (RT_FAILURE(rc))
931	return rc;
932	}
933
934	/*
935	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
936	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
937	* each time while reusing a VPID after hitting the MaxASID limit once.
938	*/
939	if ( pVM
940	&& pVM->hm.s.vmx.fVpid
941	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
942	{
943	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
944	pCpu->fFlushAsidBeforeUse = false;
945	}
946	else
947	pCpu->fFlushAsidBeforeUse = true;
948
949	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
950	++pCpu->cTlbFlushes;
951
952	return VINF_SUCCESS;
953	}
954
955
956	/**
957	* Deactivates VT-x on the current CPU.
958	*
959	* @returns VBox status code.
960	* @param pCpu Pointer to the global CPU info struct.
961	* @param pvCpuPage Pointer to the VMXON region.
962	* @param HCPhysCpuPage Physical address of the VMXON region.
963	*/
964	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
965	{
966	NOREF(pCpu);
967	NOREF(pvCpuPage);
968	NOREF(HCPhysCpuPage);
969
970	hmR0VmxLeaveRootMode();
971	return VINF_SUCCESS;
972	}
973
974
975	/**
976	* Sets the permission bits for the specified MSR in the MSR bitmap.
977	*
978	* @param pVCpu Pointer to the VMCPU.
979	* @param uMSR The MSR value.
980	* @param enmRead Whether reading this MSR causes a VM-exit.
981	* @param enmWrite Whether writing this MSR causes a VM-exit.
982	*/
983	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
984	{
985	int32_t iBit;
986	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
987
988	/*
989	* Layout:
990	* 0x000 - 0x3ff - Low MSR read bits
991	* 0x400 - 0x7ff - High MSR read bits
992	* 0x800 - 0xbff - Low MSR write bits
993	* 0xc00 - 0xfff - High MSR write bits
994	*/
995	if (uMsr <= 0x00001FFF)
996	iBit = uMsr;
997	else if ( uMsr >= 0xC0000000
998	&& uMsr <= 0xC0001FFF)
999	{
1000	iBit = (uMsr - 0xC0000000);
1001	pbMsrBitmap += 0x400;
1002	}
1003	else
1004	{
1005	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1006	return;
1007	}
1008
1009	Assert(iBit <= 0x1fff);
1010	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1011	ASMBitSet(pbMsrBitmap, iBit);
1012	else
1013	ASMBitClear(pbMsrBitmap, iBit);
1014
1015	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1016	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1017	else
1018	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1019	}
1020
1021
1022	/**
1023	* Flushes the TLB using EPT.
1024	*
1025	* @returns VBox status code.
1026	* @param pVM Pointer to the VM.
1027	* @param pVCpu Pointer to the VMCPU.
1028	* @param enmFlush Type of flush.
1029	*/
1030	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.fNestedPaging);
1034
1035	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1036
1037	uint64_t descriptor[2];
1038	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1039	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1040
1041	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1042	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1044	}
1045
1046
1047	/**
1048	* Flushes the TLB using VPID.
1049	*
1050	* @returns VBox status code.
1051	* @param pVM Pointer to the VM.
1052	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1053	* enmFlush).
1054	* @param enmFlush Type of flush.
1055	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1056	* on @a enmFlush).
1057	*/
1058	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1059	{
1060	AssertPtr(pVM);
1061	Assert(pVM->hm.s.vmx.fVpid);
1062
1063	uint64_t descriptor[2];
1064	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1065	{
1066	descriptor[0] = 0;
1067	descriptor[1] = 0;
1068	}
1069	else
1070	{
1071	AssertPtr(pVCpu);
1072	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1073	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1074	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1075	descriptor[1] = GCPtr;
1076	}
1077
1078	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1079	AssertMsg(rc == VINF_SUCCESS,
1080	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1081	if ( RT_SUCCESS(rc)
1082	&& pVCpu)
1083	{
1084	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1085	}
1086	}
1087
1088
1089	/**
1090	* Invalidates a guest page by guest virtual address. Only relevant for
1091	* EPT/VPID, otherwise there is nothing really to invalidate.
1092	*
1093	* @returns VBox status code.
1094	* @param pVM Pointer to the VM.
1095	* @param pVCpu Pointer to the VMCPU.
1096	* @param GCVirt Guest virtual address of the page to invalidate.
1097	*/
1098	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1099	{
1100	AssertPtr(pVM);
1101	AssertPtr(pVCpu);
1102	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1103
1104	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1105	if (!fFlushPending)
1106	{
1107	/*
1108	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1109	* See @bugref{6043} and @bugref{6177}.
1110	*
1111	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1112	* function maybe called in a loop with individual addresses.
1113	*/
1114	if (pVM->hm.s.vmx.fVpid)
1115	{
1116	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1117	{
1118	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1120	}
1121	else
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	}
1124	else if (pVM->hm.s.fNestedPaging)
1125	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1126	}
1127
1128	return VINF_SUCCESS;
1129	}
1130
1131
1132	/**
1133	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1134	* otherwise there is nothing really to invalidate.
1135	*
1136	* @returns VBox status code.
1137	* @param pVM Pointer to the VM.
1138	* @param pVCpu Pointer to the VMCPU.
1139	* @param GCPhys Guest physical address of the page to invalidate.
1140	*/
1141	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1142	{
1143	LogFlowFunc(("%RGp\n", GCPhys));
1144
1145	/*
1146	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1147	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1148	* This function might be called in a loop.
1149	*/
1150	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1151	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1152	return VINF_SUCCESS;
1153	}
1154
1155
1156	/**
1157	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1158	* case where neither EPT nor VPID is supported by the CPU.
1159	*
1160	* @param pVM Pointer to the VM.
1161	* @param pVCpu Pointer to the VMCPU.
1162	*
1163	* @remarks Called with interrupts disabled.
1164	*/
1165	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1166	{
1167	NOREF(pVM);
1168	AssertPtr(pVCpu);
1169	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1170	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1171
1172	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1173	AssertPtr(pCpu);
1174
1175	pVCpu->hm.s.TlbShootdown.cPages = 0;
1176	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1178	pVCpu->hm.s.fForceTLBFlush = false;
1179	return;
1180	}
1181
1182
1183	/**
1184	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1185	*
1186	* @param pVM Pointer to the VM.
1187	* @param pVCpu Pointer to the VMCPU.
1188	* @remarks All references to "ASID" in this function pertains to "VPID" in
1189	* Intel's nomenclature. The reason is, to avoid confusion in compare
1190	* statements since the host-CPU copies are named "ASID".
1191	*
1192	* @remarks Called with interrupts disabled.
1193	*/
1194	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1195	{
1196	AssertPtr(pVM);
1197	AssertPtr(pVCpu);
1198	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1199	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1200	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1201
1202	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1203	AssertPtr(pCpu);
1204
1205	/*
1206	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1207	* This can happen both for start & resume due to long jumps back to ring-3.
1208	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1209	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1210	*/
1211	bool fNewASID = false;
1212	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1213	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1214	{
1215	pVCpu->hm.s.fForceTLBFlush = true;
1216	fNewASID = true;
1217	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1218	}
1219
1220	/*
1221	* Check for explicit TLB shootdowns.
1222	*/
1223	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1224	{
1225	pVCpu->hm.s.fForceTLBFlush = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1227	}
1228
1229	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1230	if (pVCpu->hm.s.fForceTLBFlush)
1231	{
1232	if (fNewASID)
1233	{
1234	++pCpu->uCurrentAsid;
1235	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1236	{
1237	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1238	pCpu->cTlbFlushes++;
1239	pCpu->fFlushAsidBeforeUse = true;
1240	}
1241
1242	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1243	if (pCpu->fFlushAsidBeforeUse)
1244	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1245	}
1246	else
1247	{
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1249	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1250	else
1251	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1252	}
1253
1254	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1255	pVCpu->hm.s.fForceTLBFlush = false;
1256	}
1257	else
1258	{
1259	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1260	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1261	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1262	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1263
1264	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1265	* not be executed. See hmQueueInvlPage() where it is commented
1266	* out. Support individual entry flushing someday. */
1267	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1268	{
1269	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1270
1271	/*
1272	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1273	* as supported by the CPU.
1274	*/
1275	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1276	{
1277	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1278	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1279	}
1280	else
1281	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1282	}
1283	else
1284	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1285	}
1286	pVCpu->hm.s.TlbShootdown.cPages = 0;
1287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1288
1289	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1290	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1291	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1292	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1293	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1294	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1295
1296	/* Update VMCS with the VPID. */
1297	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1298	AssertRC(rc);
1299	}
1300
1301
1302	/**
1303	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1304	*
1305	* @returns VBox status code.
1306	* @param pVM Pointer to the VM.
1307	* @param pVCpu Pointer to the VMCPU.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1312	{
1313	AssertPtr(pVM);
1314	AssertPtr(pVCpu);
1315	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1316	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1317
1318	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1319	AssertPtr(pCpu);
1320
1321	/*
1322	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1323	* This can happen both for start & resume due to long jumps back to ring-3.
1324	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1325	*/
1326	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1327	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1328	{
1329	pVCpu->hm.s.fForceTLBFlush = true;
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1331	}
1332
1333	/* Check for explicit TLB shootdown flushes. */
1334	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1335	{
1336	pVCpu->hm.s.fForceTLBFlush = true;
1337	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1338	}
1339
1340	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343	if (pVCpu->hm.s.fForceTLBFlush)
1344	{
1345	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1346	pVCpu->hm.s.fForceTLBFlush = false;
1347	}
1348	else
1349	{
1350	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1351	* not be executed. See hmQueueInvlPage() where it is commented
1352	* out. Support individual entry flushing someday. */
1353	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1354	{
1355	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1357	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1358	}
1359	else
1360	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1361	}
1362
1363	pVCpu->hm.s.TlbShootdown.cPages = 0;
1364	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1365	}
1366
1367
1368	/**
1369	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1370	*
1371	* @returns VBox status code.
1372	* @param pVM Pointer to the VM.
1373	* @param pVCpu Pointer to the VMCPU.
1374	*
1375	* @remarks Called with interrupts disabled.
1376	*/
1377	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1378	{
1379	AssertPtr(pVM);
1380	AssertPtr(pVCpu);
1381	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1382	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1383
1384	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1385
1386	/*
1387	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1388	* This can happen both for start & resume due to long jumps back to ring-3.
1389	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1390	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1391	*/
1392	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1393	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1394	{
1395	pVCpu->hm.s.fForceTLBFlush = true;
1396	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1397	}
1398
1399	/* Check for explicit TLB shootdown flushes. */
1400	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1401	{
1402	/*
1403	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1404	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1405	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1406	*/
1407	pVCpu->hm.s.fForceTLBFlush = true;
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1409	}
1410
1411	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1412	if (pVCpu->hm.s.fForceTLBFlush)
1413	{
1414	++pCpu->uCurrentAsid;
1415	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1416	{
1417	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1418	pCpu->fFlushAsidBeforeUse = true;
1419	pCpu->cTlbFlushes++;
1420	}
1421
1422	pVCpu->hm.s.fForceTLBFlush = false;
1423	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1424	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1425	if (pCpu->fFlushAsidBeforeUse)
1426	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1427	}
1428	else
1429	{
1430	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1431	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1432	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1433	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1434
1435	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1436	* not be executed. See hmQueueInvlPage() where it is commented
1437	* out. Support individual entry flushing someday. */
1438	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1439	{
1440	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1441	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1442	{
1443	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1444	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1445	}
1446	else
1447	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1448	}
1449	else
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1451	}
1452
1453	pVCpu->hm.s.TlbShootdown.cPages = 0;
1454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1455
1456	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1457	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1458	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1459	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1460	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1461	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1462
1463	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1464	AssertRC(rc);
1465	}
1466
1467
1468	/**
1469	* Flushes the guest TLB entry based on CPU capabilities.
1470	*
1471	* @param pVCpu Pointer to the VMCPU.
1472	*/
1473	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1474	{
1475	PVM pVM = pVCpu->CTX_SUFF(pVM);
1476	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1477	{
1478	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1479	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1480	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1481	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1482	default:
1483	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1484	break;
1485	}
1486	}
1487
1488
1489	/**
1490	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1491	* TLB entries from the host TLB before VM-entry.
1492	*
1493	* @returns VBox status code.
1494	* @param pVM Pointer to the VM.
1495	*/
1496	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1497	{
1498	/*
1499	* Determine optimal flush type for nested paging.
1500	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1501	* guest execution (see hmR3InitFinalizeR0()).
1502	*/
1503	if (pVM->hm.s.fNestedPaging)
1504	{
1505	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1506	{
1507	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1508	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1509	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1510	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1511	else
1512	{
1513	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1514	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1515	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1516	}
1517
1518	/* Make sure the write-back cacheable memory type for EPT is supported. */
1519	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1520	{
1521	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1522	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1523	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1524	}
1525	}
1526	else
1527	{
1528	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1529	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1531	}
1532	}
1533
1534	/*
1535	* Determine optimal flush type for VPID.
1536	*/
1537	if (pVM->hm.s.vmx.fVpid)
1538	{
1539	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1540	{
1541	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1542	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1543	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1544	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1545	else
1546	{
1547	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1549	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1551	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1552	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1553	pVM->hm.s.vmx.fVpid = false;
1554	}
1555	}
1556	else
1557	{
1558	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1559	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1560	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1561	pVM->hm.s.vmx.fVpid = false;
1562	}
1563	}
1564
1565	/*
1566	* Setup the handler for flushing tagged-TLBs.
1567	*/
1568	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1569	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1570	else if (pVM->hm.s.fNestedPaging)
1571	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1572	else if (pVM->hm.s.vmx.fVpid)
1573	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1574	else
1575	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1576	return VINF_SUCCESS;
1577	}
1578
1579
1580	/**
1581	* Sets up pin-based VM-execution controls in the VMCS.
1582	*
1583	* @returns VBox status code.
1584	* @param pVM Pointer to the VM.
1585	* @param pVCpu Pointer to the VMCPU.
1586	*/
1587	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1588	{
1589	AssertPtr(pVM);
1590	AssertPtr(pVCpu);
1591
1592	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1593	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1594
1595	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1596	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1597	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1598
1599	/* Enable the VMX preemption timer. */
1600	if (pVM->hm.s.vmx.fUsePreemptTimer)
1601	{
1602	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1603	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1604	}
1605
1606	if ((val & zap) != val)
1607	{
1608	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1609	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1610	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1611	}
1612
1613	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1614	AssertRCReturn(rc, rc);
1615
1616	/* Update VCPU with the currently set pin-based VM-execution controls. */
1617	pVCpu->hm.s.vmx.u32PinCtls = val;
1618	return rc;
1619	}
1620
1621
1622	/**
1623	* Sets up processor-based VM-execution controls in the VMCS.
1624	*
1625	* @returns VBox status code.
1626	* @param pVM Pointer to the VM.
1627	* @param pVMCPU Pointer to the VMCPU.
1628	*/
1629	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1630	{
1631	AssertPtr(pVM);
1632	AssertPtr(pVCpu);
1633
1634	int rc = VERR_INTERNAL_ERROR_5;
1635	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1636	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1637
1638	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1639	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1640	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1641	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1645
1646	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1647	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1648	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1649	{
1650	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1652	}
1653
1654	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1655	if (!pVM->hm.s.fNestedPaging)
1656	{
1657	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1658	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1659	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1660	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1661	}
1662
1663	/* Use TPR shadowing if supported by the CPU. */
1664	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1665	{
1666	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1667	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1668	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1669	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1670	AssertRCReturn(rc, rc);
1671
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1673	/* CR8 writes causes a VM-exit based on TPR threshold. */
1674	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1675	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1676	}
1677	else
1678	{
1679	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1681	}
1682
1683	/* Use MSR-bitmaps if supported by the CPU. */
1684	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1685	{
1686	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1687
1688	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1689	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1690	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1691	AssertRCReturn(rc, rc);
1692
1693	/*
1694	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1695	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1696	*/
1697	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1698	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1699	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1700	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1701	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1702	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1703	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1704	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1705	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1706	}
1707
1708	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1710	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1715	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls = val;
1724
1725	/*
1726	* Secondary processor-based VM-execution controls.
1727	*/
1728	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1729	{
1730	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1731	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1732
1733	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1734	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1735
1736	if (pVM->hm.s.fNestedPaging)
1737	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1738	else
1739	{
1740	/*
1741	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1742	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1743	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1744	*/
1745	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1747	}
1748
1749	if (pVM->hm.s.vmx.fVpid)
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1751
1752	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1753	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1754
1755	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1756	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1757	* done dynamically. */
1758	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1759	{
1760	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1761	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1763	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1764	AssertRCReturn(rc, rc);
1765	}
1766
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1768	{
1769	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1771	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1772	}
1773
1774	if ((val & zap) != val)
1775	{
1776	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1777	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1778	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1779	}
1780
1781	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1782	AssertRCReturn(rc, rc);
1783
1784	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1785	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1786	}
1787
1788	return VINF_SUCCESS;
1789	}
1790
1791
1792	/**
1793	* Sets up miscellaneous (everything other than Pin & Processor-based
1794	* VM-execution) control fields in the VMCS.
1795	*
1796	* @returns VBox status code.
1797	* @param pVM Pointer to the VM.
1798	* @param pVCpu Pointer to the VMCPU.
1799	*/
1800	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1801	{
1802	AssertPtr(pVM);
1803	AssertPtr(pVCpu);
1804
1805	int rc = VERR_GENERAL_FAILURE;
1806
1807	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1808	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1809
1810	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1811
1812	/*
1813	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1814	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1815	* We thus use the exception bitmap to control it rather than use both.
1816	*/
1817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1818	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1819
1820	/** @todo Explore possibility of using IO-bitmaps. */
1821	/* All IO & IOIO instructions cause VM-exits. */
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1823	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1824
1825	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1826	/* Setup MSR autoloading/autostoring. */
1827	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1828	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1830	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1831	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1832	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1833
1834	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1835	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1836	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1837	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1838	#else
1839	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1840	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1841	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1842	#endif
1843
1844	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1845	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1846
1847	/* Setup debug controls */
1848	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1849	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1850	AssertRCReturn(rc, rc);
1851	return rc;
1852	}
1853
1854
1855	/**
1856	* Sets up the initial exception bitmap in the VMCS based on static conditions
1857	* (i.e. conditions that cannot ever change at runtime).
1858	*
1859	* @returns VBox status code.
1860	* @param pVM Pointer to the VM.
1861	* @param pVCpu Pointer to the VMCPU.
1862	*/
1863	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1864	{
1865	AssertPtr(pVM);
1866	AssertPtr(pVCpu);
1867
1868	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1869
1870	uint32_t u32XcptBitmap = 0;
1871
1872	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1873	if (!pVM->hm.s.fNestedPaging)
1874	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1875
1876	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1877	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1878	AssertRCReturn(rc, rc);
1879	return rc;
1880	}
1881
1882
1883	/**
1884	* Sets up the initial guest-state mask. The guest-state mask is consulted
1885	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1886	* for the nested virtualization case (as it would cause a VM-exit).
1887	*
1888	* @param pVCpu Pointer to the VMCPU.
1889	*/
1890	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1891	{
1892	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1893	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1894	return VINF_SUCCESS;
1895	}
1896
1897
1898	/**
1899	* Does per-VM VT-x initialization.
1900	*
1901	* @returns VBox status code.
1902	* @param pVM Pointer to the VM.
1903	*/
1904	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1905	{
1906	LogFlowFunc(("pVM=%p\n", pVM));
1907
1908	int rc = hmR0VmxStructsAlloc(pVM);
1909	if (RT_FAILURE(rc))
1910	{
1911	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1912	return rc;
1913	}
1914
1915	return VINF_SUCCESS;
1916	}
1917
1918
1919	/**
1920	* Does per-VM VT-x termination.
1921	*
1922	* @returns VBox status code.
1923	* @param pVM Pointer to the VM.
1924	*/
1925	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1926	{
1927	LogFlowFunc(("pVM=%p\n", pVM));
1928
1929	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1930	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1931	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1932	#endif
1933	hmR0VmxStructsFree(pVM);
1934	return VINF_SUCCESS;
1935	}
1936
1937
1938	/**
1939	* Sets up the VM for execution under VT-x.
1940	* This function is only called once per-VM during initalization.
1941	*
1942	* @returns VBox status code.
1943	* @param pVM Pointer to the VM.
1944	*/
1945	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1946	{
1947	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1948	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1949
1950	LogFlowFunc(("pVM=%p\n", pVM));
1951
1952	/*
1953	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1954	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1955	*/
1956	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1957	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1958	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1959	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1960	{
1961	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1962	return VERR_INTERNAL_ERROR;
1963	}
1964
1965	/* Initialize these always, see hmR3InitFinalizeR0().*/
1966	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1967	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1968
1969	/* Setup the tagged-TLB flush handlers. */
1970	int rc = hmR0VmxSetupTaggedTlb(pVM);
1971	if (RT_FAILURE(rc))
1972	{
1973	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1974	return rc;
1975	}
1976
1977	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1978	{
1979	PVMCPU pVCpu = &pVM->aCpus[i];
1980	AssertPtr(pVCpu);
1981	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1982
1983	/* Set revision dword at the beginning of the VMCS structure. */
1984	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1985
1986	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1987	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1988	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1989	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1990
1991	/* Load this VMCS as the current VMCS. */
1992	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1993	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1994	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1995
1996	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1997	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1998	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1999
2000	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2001	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2002	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2003
2004	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2017	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2018	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2019	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2020	#endif
2021
2022	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2023	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2024	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2025	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2026
2027	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2028	}
2029
2030	return VINF_SUCCESS;
2031	}
2032
2033
2034	/**
2035	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2036	* the VMCS.
2037	*
2038	* @returns VBox status code.
2039	* @param pVM Pointer to the VM.
2040	* @param pVCpu Pointer to the VMCPU.
2041	*/
2042	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2043	{
2044	RTCCUINTREG uReg = ASMGetCR0();
2045	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2046
2047	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2048	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2049	if (HMVMX_IS_64BIT_HOST_MODE())
2050	{
2051	uint64_t uRegCR3 = hmR0Get64bitCR3();
2052	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2053	}
2054	else
2055	#endif
2056	{
2057	uReg = ASMGetCR3();
2058	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2059	}
2060
2061	uReg = ASMGetCR4();
2062	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2063	AssertRCReturn(rc, rc);
2064	return rc;
2065	}
2066
2067
2068	/**
2069	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2070	* the host-state area in the VMCS.
2071	*
2072	* @returns VBox status code.
2073	* @param pVM Pointer to the VM.
2074	* @param pVCpu Pointer to the VMCPU.
2075	*/
2076	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2077	{
2078	int rc = VERR_INTERNAL_ERROR_5;
2079	RTSEL uSelCS = 0;
2080	RTSEL uSelSS = 0;
2081	RTSEL uSelDS = 0;
2082	RTSEL uSelES = 0;
2083	RTSEL uSelFS = 0;
2084	RTSEL uSelGS = 0;
2085	RTSEL uSelTR = 0;
2086
2087	/*
2088	* Host Selector registers.
2089	*/
2090	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2091	if (HMVMX_IS_64BIT_HOST_MODE())
2092	{
2093	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2094	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2095	}
2096	else
2097	{
2098	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2099	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2100	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2101	}
2102	#else
2103	uSelCS = ASMGetCS();
2104	uSelSS = ASMGetSS();
2105	#endif
2106
2107	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2108	/** @todo Verify if we have any platform that actually run with DS or ES with
2109	* RPL != 0 in kernel space. */
2110	uSelDS = 0;
2111	uSelES = 0;
2112	uSelFS = 0;
2113	uSelGS = 0;
2114	uSelTR = ASMGetTR();
2115
2116	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2117	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2118	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2119	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2120	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2121	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2122	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2123	Assert(uSelCS != 0);
2124	Assert(uSelTR != 0);
2125
2126	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2127	#if 0
2128	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2129	Assert(uSelSS != 0);
2130	#endif
2131
2132	/* Write these host selector fields into the host-state area in the VMCS. */
2133	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2134	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2135	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2136	#if 0
2137	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2138	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2139	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2140	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2141	#endif
2142	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2143	AssertRCReturn(rc, rc);
2144
2145	/*
2146	* Host GDTR and IDTR.
2147	*/
2148	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2149	* be safe to -not- save and restore GDTR and IDTR in the assembly
2150	* code and just do it here and don't care if the limits are zapped on
2151	* VM-exit. */
2152	RTGDTR Gdtr;
2153	RT_ZERO(Gdtr);
2154	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2155	if (HMVMX_IS_64BIT_HOST_MODE())
2156	{
2157	X86XDTR64 Gdtr64;
2158	X86XDTR64 Idtr64;
2159	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2160	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2161	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2162	Gdtr.cbGdt = Gdtr64.cb;
2163	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2164	}
2165	else
2166	#endif
2167	{
2168	RTIDTR Idtr;
2169	ASMGetGDTR(&Gdtr);
2170	ASMGetIDTR(&Idtr);
2171	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2172	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2173	}
2174	AssertRCReturn(rc, rc);
2175
2176	/*
2177	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2178	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2179	*/
2180	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2181	{
2182	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2183	return VERR_VMX_INVALID_HOST_STATE;
2184	}
2185
2186	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2187	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2188	if (HMVMX_IS_64BIT_HOST_MODE())
2189	{
2190	/* We need the 64-bit TR base for hybrid darwin. */
2191	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2192	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2193	}
2194	else
2195	#endif
2196	{
2197	uintptr_t uTRBase;
2198	#if HC_ARCH_BITS == 64
2199	uTRBase = X86DESC64_BASE(pDesc);
2200	#else
2201	uTRBase = X86DESC_BASE(pDesc);
2202	#endif
2203	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2204	}
2205	AssertRCReturn(rc, rc);
2206
2207	/*
2208	* Host FS base and GS base.
2209	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2210	* would take care of the bases. In 64-bit, the MSRs come into play.
2211	*/
2212	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2213	if (HMVMX_IS_64BIT_HOST_MODE())
2214	{
2215	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2216	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2217	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2218	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2219	AssertRCReturn(rc, rc);
2220	}
2221	#endif
2222	return rc;
2223	}
2224
2225
2226	/**
2227	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2228	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2229	* the host after every successful VM exit.
2230	*
2231	* @returns VBox status code.
2232	* @param pVM Pointer to the VM.
2233	* @param pVCpu Pointer to the VMCPU.
2234	*/
2235	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2236	{
2237	AssertPtr(pVCpu);
2238	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2239
2240	int rc = VINF_SUCCESS;
2241	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2242	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2243	uint32_t cHostMsrs = 0;
2244	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2245
2246	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2247	{
2248	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2249	pHostMsr->u32Reserved = 0;
2250	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2251	if (CPUMIsGuestInLongMode(pVCpu))
2252	{
2253	/* Must match the EFER value in our 64 bits switcher. */
2254	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2255	}
2256	else
2257	# endif
2258	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2259	pHostMsr++; cHostMsrs++;
2260	}
2261
2262	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2263	if (HMVMX_IS_64BIT_HOST_MODE())
2264	{
2265	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2266	pHostMsr->u32Reserved = 0;
2267	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2268	pHostMsr++; cHostMsrs++;
2269	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2270	pHostMsr->u32Reserved = 0;
2271	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2272	pHostMsr++; cHostMsrs++;
2273	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2274	pHostMsr->u32Reserved = 0;
2275	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2276	pHostMsr++; cHostMsrs++;
2277	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2278	pHostMsr->u32Reserved = 0;
2279	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2280	pHostMsr++; cHostMsrs++;
2281	}
2282	# endif
2283
2284	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2285	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2286	{
2287	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2288	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2289	}
2290
2291	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2292	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2293
2294	/*
2295	* Host Sysenter MSRs.
2296	*/
2297	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2298	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2299	if (HMVMX_IS_64BIT_HOST_MODE())
2300	{
2301	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2302	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2303	}
2304	else
2305	{
2306	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2308	}
2309	# elif HC_ARCH_BITS == 32
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2311	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2312	# else
2313	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2314	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2315	# endif
2316	AssertRCReturn(rc, rc);
2317
2318	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2319	* hmR0VmxSetupExitCtls() !! */
2320	return rc;
2321	}
2322
2323
2324	/**
2325	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2326	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2327	* controls".
2328	*
2329	* @returns VBox status code.
2330	* @param pVCpu Pointer to the VMCPU.
2331	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2332	* out-of-sync. Make sure to update the required fields
2333	* before using them.
2334	*
2335	* @remarks No-long-jump zone!!!
2336	*/
2337	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2338	{
2339	int rc = VINF_SUCCESS;
2340	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2341	{
2342	PVM pVM = pVCpu->CTX_SUFF(pVM);
2343	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2344	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2345
2346	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2347	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2348
2349	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2350	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2351	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2352	else
2353	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2354
2355	/*
2356	* The following should not be set (since we're not in SMM mode):
2357	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2358	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2359	*/
2360
2361	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2362	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2363	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2364
2365	if ((val & zap) != val)
2366	{
2367	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2368	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2369	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2370	}
2371
2372	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2373	AssertRCReturn(rc, rc);
2374
2375	/* Update VCPU with the currently set VM-exit controls. */
2376	pVCpu->hm.s.vmx.u32EntryCtls = val;
2377	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2378	}
2379	return rc;
2380	}
2381
2382
2383	/**
2384	* Sets up the VM-exit controls in the VMCS.
2385	*
2386	* @returns VBox status code.
2387	* @param pVM Pointer to the VM.
2388	* @param pVCpu Pointer to the VMCPU.
2389	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2390	* out-of-sync. Make sure to update the required fields
2391	* before using them.
2392	*
2393	* @remarks requires EFER.
2394	*/
2395	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2396	{
2397	int rc = VINF_SUCCESS;
2398	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2399	{
2400	PVM pVM = pVCpu->CTX_SUFF(pVM);
2401	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2402	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2403
2404	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2405	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2406
2407	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2408	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2409	if (HMVMX_IS_64BIT_HOST_MODE())
2410	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2411	else
2412	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2413	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2414	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2415	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2416	else
2417	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2418	#endif
2419
2420	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2421	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2422
2423	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2424	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2425	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2426	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2427	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2428
2429	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2430	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2431
2432	if ((val & zap) != val)
2433	{
2434	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2435	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2436	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2437	}
2438
2439	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2440	AssertRCReturn(rc, rc);
2441
2442	/* Update VCPU with the currently set VM-exit controls. */
2443	pVCpu->hm.s.vmx.u32ExitCtls = val;
2444	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2445	}
2446	return rc;
2447	}
2448
2449
2450	/**
2451	* Loads the guest APIC and related state.
2452	*
2453	* @returns VBox status code.
2454	* @param pVM Pointer to the VM.
2455	* @param pVCpu Pointer to the VMCPU.
2456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2457	* out-of-sync. Make sure to update the required fields
2458	* before using them.
2459	*/
2460	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2461	{
2462	int rc = VINF_SUCCESS;
2463	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2464	{
2465	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2466	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2467	{
2468	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2469
2470	bool fPendingIntr = false;
2471	uint8_t u8GuestTpr = 0;
2472	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2473	AssertRCReturn(rc, rc);
2474
2475	/*
2476	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2477	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2478	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2479	* the interrupt when we VM-exit for other reasons.
2480	*/
2481	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2482	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2483	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2484	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2485
2486	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2487	AssertRCReturn(rc, rc);
2488
2489	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2490	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2491	{
2492	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2493	pMixedCtx->msrLSTAR = u8GuestTpr;
2494	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2495	{
2496	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2497	if (fPendingIntr)
2498	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2499	else
2500	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2501	}
2502	}
2503	}
2504
2505	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2506	}
2507	return rc;
2508	}
2509
2510
2511	/**
2512	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2513	*
2514	* @returns
2515	* @param pVCpu Pointer to the VMCPU.
2516	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2517	* out-of-sync. Make sure to update the required fields
2518	* before using them.
2519	*
2520	* @remarks No-long-jump zone!!!
2521	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2522	*/
2523	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2524	{
2525	/*
2526	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2527	* inhibit interrupts or clear any existing interrupt-inhibition.
2528	*/
2529	uint32_t uIntrState = 0;
2530	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2531	{
2532	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2533	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2534	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2535	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2536	{
2537	/*
2538	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2539	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2540	*/
2541	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2542	}
2543	else if (pMixedCtx->eflags.Bits.u1IF)
2544	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2545	else
2546	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2547	}
2548	return uIntrState;
2549	}
2550
2551
2552	/**
2553	* Loads the guest's interruptibility-state into the guest-state area in the
2554	* VMCS.
2555	*
2556	* @returns VBox status code.
2557	* @param pVCpu Pointer to the VMCPU.
2558	* @param uIntrState The interruptibility-state to set.
2559	*/
2560	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2561	{
2562	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2563	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2564	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2565	AssertRCReturn(rc, rc);
2566	return rc;
2567	}
2568
2569
2570	/**
2571	* Loads the guest's RIP into the guest-state area in the VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @param pVCpu Pointer to the VMCPU.
2575	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2576	* out-of-sync. Make sure to update the required fields
2577	* before using them.
2578	*
2579	* @remarks No-long-jump zone!!!
2580	*/
2581	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2582	{
2583	int rc = VINF_SUCCESS;
2584	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2585	{
2586	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2587	AssertRCReturn(rc, rc);
2588	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2589	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2590	}
2591	return rc;
2592	}
2593
2594
2595	/**
2596	* Loads the guest's RSP into the guest-state area in the VMCS.
2597	*
2598	* @returns VBox status code.
2599	* @param pVCpu Pointer to the VMCPU.
2600	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2601	* out-of-sync. Make sure to update the required fields
2602	* before using them.
2603	*
2604	* @remarks No-long-jump zone!!!
2605	*/
2606	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2607	{
2608	int rc = VINF_SUCCESS;
2609	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2610	{
2611	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2612	AssertRCReturn(rc, rc);
2613	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2614	}
2615	return rc;
2616	}
2617
2618
2619	/**
2620	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2621	*
2622	* @returns VBox status code.
2623	* @param pVCpu Pointer to the VMCPU.
2624	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2625	* out-of-sync. Make sure to update the required fields
2626	* before using them.
2627	*
2628	* @remarks No-long-jump zone!!!
2629	*/
2630	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2631	{
2632	int rc = VINF_SUCCESS;
2633	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2634	{
2635	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2636	Let us assert it as such and use 32-bit VMWRITE. */
2637	Assert(!(pMixedCtx->rflags.u64 >> 32));
2638	X86EFLAGS uEFlags = pMixedCtx->eflags;
2639	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2640	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2641
2642	/*
2643	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2644	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2645	*/
2646	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2647	{
2648	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2649	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2650	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2651	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2652	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2653	}
2654
2655	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2656	AssertRCReturn(rc, rc);
2657
2658	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2659	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2660	}
2661	return rc;
2662	}
2663
2664
2665	/**
2666	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2667	*
2668	* @returns VBox status code.
2669	* @param pVCpu Pointer to the VMCPU.
2670	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2671	* out-of-sync. Make sure to update the required fields
2672	* before using them.
2673	*
2674	* @remarks No-long-jump zone!!!
2675	*/
2676	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2677	{
2678	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2679	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2680	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2681	return rc;
2682	}
2683
2684
2685	/**
2686	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2687	* in the VMCS.
2688	*
2689	* @returns VBox status code.
2690	* @param pVM Pointer to the VM.
2691	* @param pVCpu Pointer to the VMCPU.
2692	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2693	* out-of-sync. Make sure to update the required fields
2694	* before using them.
2695	*
2696	* @remarks No-long-jump zone!!!
2697	*/
2698	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2699	{
2700	int rc = VINF_SUCCESS;
2701	PVM pVM = pVCpu->CTX_SUFF(pVM);
2702
2703	/*
2704	* Guest CR0.
2705	* Guest FPU.
2706	*/
2707	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2708	{
2709	Assert(!(pCtx->cr0 >> 32));
2710	uint32_t u32GuestCR0 = pCtx->cr0;
2711
2712	/* The guest's view (read access) of its CR0 is unblemished. */
2713	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2714	AssertRCReturn(rc, rc);
2715	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2716
2717	/* Setup VT-x's view of the guest CR0. */
2718	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2719	if (pVM->hm.s.fNestedPaging)
2720	{
2721	if (CPUMIsGuestPagingEnabledEx(pCtx))
2722	{
2723	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2724	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2725	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2726	}
2727	else
2728	{
2729	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2730	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2731	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2732	}
2733
2734	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2735	AssertRCReturn(rc, rc);
2736	}
2737	else
2738	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2739
2740	/*
2741	* Guest FPU bits.
2742	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2743	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2744	*/
2745	u32GuestCR0 \|= X86_CR0_NE;
2746	bool fInterceptNM = false;
2747	if (CPUMIsGuestFPUStateActive(pVCpu))
2748	{
2749	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2750	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2751	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2752	}
2753	else
2754	{
2755	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2756	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2757	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2758	}
2759
2760	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2761	bool fInterceptMF = false;
2762	if (!(pCtx->cr0 & X86_CR0_NE))
2763	fInterceptMF = true;
2764
2765	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2766	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2767	{
2768	Assert(PDMVmmDevHeapIsEnabled(pVM));
2769	Assert(pVM->hm.s.vmx.pRealModeTSS);
2770	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2771	fInterceptNM = true;
2772	fInterceptMF = true;
2773	}
2774	else
2775	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2776
2777	if (fInterceptNM)
2778	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2779	else
2780	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2781
2782	if (fInterceptMF)
2783	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2784	else
2785	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2786
2787	/* Additional intercepts for debugging, define these yourself explicitly. */
2788	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2789	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2790	\| RT_BIT(X86_XCPT_DB)
2791	\| RT_BIT(X86_XCPT_DE)
2792	\| RT_BIT(X86_XCPT_NM)
2793	\| RT_BIT(X86_XCPT_UD)
2794	\| RT_BIT(X86_XCPT_NP)
2795	\| RT_BIT(X86_XCPT_SS)
2796	\| RT_BIT(X86_XCPT_GP)
2797	\| RT_BIT(X86_XCPT_PF)
2798	\| RT_BIT(X86_XCPT_MF);
2799	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2800	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2801	#endif
2802
2803	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2804
2805	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2806	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2807	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2808	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2809	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2810	else
2811	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2812
2813	u32GuestCR0 \|= uSetCR0;
2814	u32GuestCR0 &= uZapCR0;
2815	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2816
2817	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2818	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2820	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2821
2822	/*
2823	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2824	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2825	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2826	*/
2827	uint64_t u64CR0Mask = 0;
2828	u64CR0Mask = X86_CR0_PE
2829	\| X86_CR0_NE
2830	\| X86_CR0_WP
2831	\| X86_CR0_PG
2832	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2833	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2834	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2835	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2836	u64CR0Mask &= ~X86_CR0_PE;
2837	if (pVM->hm.s.fNestedPaging)
2838	u64CR0Mask &= ~X86_CR0_WP;
2839
2840	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2841	if (fInterceptNM)
2842	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2843	else
2844	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2845
2846	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2847	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2848	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2849	AssertRCReturn(rc, rc);
2850
2851	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2852	}
2853
2854	/*
2855	* Guest CR2.
2856	* It's always loaded in the assembler code. Nothing to do here.
2857	*/
2858
2859	/*
2860	* Guest CR3.
2861	*/
2862	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2863	{
2864	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2865	if (pVM->hm.s.fNestedPaging)
2866	{
2867	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2868
2869	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2870	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2871	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2872	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2873
2874	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2875	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2876	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2877
2878	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2879	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2880	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2881	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2882
2883	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2884	AssertRCReturn(rc, rc);
2885	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2886
2887	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2888	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2889	{
2890	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2891	if (CPUMIsGuestInPAEModeEx(pCtx))
2892	{
2893	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2894	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2895	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2896	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2897	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2898	AssertRCReturn(rc, rc);
2899	}
2900
2901	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2902	have Unrestricted Execution to handle the guest when it's not using paging. */
2903	GCPhysGuestCR3 = pCtx->cr3;
2904	}
2905	else
2906	{
2907	/*
2908	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2909	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2910	* EPT takes care of translating it to host-physical addresses.
2911	*/
2912	RTGCPHYS GCPhys;
2913	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2914	Assert(PDMVmmDevHeapIsEnabled(pVM));
2915
2916	/* We obtain it here every time as the guest could have relocated this PCI region. */
2917	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2918	AssertRCReturn(rc, rc);
2919
2920	GCPhysGuestCR3 = GCPhys;
2921	}
2922	}
2923	else
2924	{
2925	/* Non-nested paging case, just use the hypervisor's CR3. */
2926	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2927	}
2928
2929	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2930	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2931	AssertRCReturn(rc, rc);
2932
2933	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2934	}
2935
2936	/*
2937	* Guest CR4.
2938	*/
2939	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2940	{
2941	Assert(!(pCtx->cr4 >> 32));
2942	uint32_t u32GuestCR4 = pCtx->cr4;
2943
2944	/* The guest's view of its CR4 is unblemished. */
2945	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2946	AssertRCReturn(rc, rc);
2947	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2948
2949	/* Setup VT-x's view of the guest CR4. */
2950	/*
2951	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2952	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2953	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2954	*/
2955	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2956	{
2957	Assert(pVM->hm.s.vmx.pRealModeTSS);
2958	Assert(PDMVmmDevHeapIsEnabled(pVM));
2959	u32GuestCR4 &= ~X86_CR4_VME;
2960	}
2961
2962	if (pVM->hm.s.fNestedPaging)
2963	{
2964	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2965	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2966	{
2967	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2968	u32GuestCR4 \|= X86_CR4_PSE;
2969	/* Our identity mapping is a 32 bits page directory. */
2970	u32GuestCR4 &= ~X86_CR4_PAE;
2971	}
2972	/* else use guest CR4.*/
2973	}
2974	else
2975	{
2976	/*
2977	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2978	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2979	*/
2980	switch (pVCpu->hm.s.enmShadowMode)
2981	{
2982	case PGMMODE_REAL: /* Real-mode. */
2983	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2984	case PGMMODE_32_BIT: /* 32-bit paging. */
2985	{
2986	u32GuestCR4 &= ~X86_CR4_PAE;
2987	break;
2988	}
2989
2990	case PGMMODE_PAE: /* PAE paging. */
2991	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2992	{
2993	u32GuestCR4 \|= X86_CR4_PAE;
2994	break;
2995	}
2996
2997	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2998	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2999	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3000	break;
3001	#endif
3002	default:
3003	AssertFailed();
3004	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3005	}
3006	}
3007
3008	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3009	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3010	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3011	u32GuestCR4 \|= uSetCR4;
3012	u32GuestCR4 &= uZapCR4;
3013
3014	/* Write VT-x's view of the guest CR4 into the VMCS. */
3015	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3016	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3017
3018	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3019	uint64_t u64CR4Mask = 0;
3020	u64CR4Mask = X86_CR4_VME
3021	\| X86_CR4_PAE
3022	\| X86_CR4_PGE
3023	\| X86_CR4_PSE
3024	\| X86_CR4_VMXE;
3025	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3026	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3027	AssertRCReturn(rc, rc);
3028
3029	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3030	}
3031	return rc;
3032	}
3033
3034
3035	/**
3036	* Loads the guest debug registers into the guest-state area in the VMCS.
3037	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3038	*
3039	* @returns VBox status code.
3040	* @param pVCpu Pointer to the VMCPU.
3041	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3042	* out-of-sync. Make sure to update the required fields
3043	* before using them.
3044	*
3045	* @remarks No-long-jump zone!!!
3046	*/
3047	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3048	{
3049	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3050	return VINF_SUCCESS;
3051
3052	#ifdef VBOX_STRICT
3053	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3054	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3055	{
3056	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3057	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3058	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3059	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3060	}
3061	#endif
3062
3063	int rc = VERR_INTERNAL_ERROR_5;
3064	PVM pVM = pVCpu->CTX_SUFF(pVM);
3065	bool fInterceptDB = false;
3066	bool fInterceptMovDRx = false;
3067	if (DBGFIsStepping(pVCpu))
3068	{
3069	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3070	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3071	{
3072	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3073	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3074	AssertRCReturn(rc, rc);
3075	Assert(fInterceptDB == false);
3076	}
3077	else
3078	fInterceptDB = true;
3079	}
3080
3081	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3082	{
3083	if (!CPUMIsHyperDebugStateActive(pVCpu))
3084	{
3085	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3086	AssertRC(rc);
3087	}
3088	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3089	fInterceptMovDRx = true;
3090	}
3091	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3092	{
3093	if (!CPUMIsGuestDebugStateActive(pVCpu))
3094	{
3095	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3096	AssertRC(rc);
3097	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3098	}
3099	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3100	Assert(fInterceptMovDRx == false);
3101	}
3102	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3103	{
3104	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3105	fInterceptMovDRx = true;
3106	}
3107
3108	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3109	if (fInterceptDB)
3110	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3111	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3112	{
3113	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3114	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3115	#endif
3116	}
3117
3118	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3119	if (fInterceptMovDRx)
3120	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3121	else
3122	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3123
3124	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3125	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3126
3127	/* The guest's view of its DR7 is unblemished. */
3128	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3129
3130	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3131	return rc;
3132	}
3133
3134
3135	#ifdef VBOX_STRICT
3136	/**
3137	* Strict function to validate segment registers.
3138	*/
3139	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3140	{
3141	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3142	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3143	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3144	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3145	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3146	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3147	{
3148	/* Protected mode checks */
3149	/* CS */
3150	Assert(pCtx->cs.Attr.n.u1Present);
3151	Assert(!(pCtx->cs.Attr.u & 0xf00));
3152	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3153	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3154	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3155	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3156	\|\| (pCtx->cs.Attr.n.u1Granularity));
3157	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3158	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3159	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3160	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3161	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3162	else
3163	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3164	/* SS */
3165	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3166	{
3167	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3168	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3169	Assert(pCtx->ss.Attr.n.u1Present);
3170	Assert(!(pCtx->ss.Attr.u & 0xf00));
3171	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3172	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3173	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3174	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3175	\|\| (pCtx->ss.Attr.n.u1Granularity));
3176	}
3177	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3178	/* CR0 might not be up-to-date here always, hence disabled. */
3179	#if 0
3180	if (!pCtx->cr0 & X86_CR0_PE)
3181	Assert(!pCtx->ss.Attr.n.u2Dpl);
3182	#endif
3183	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3184	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3185	{
3186	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3187	Assert(pCtx->ds.Attr.n.u1Present);
3188	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3189	Assert(!(pCtx->ds.Attr.u & 0xf00));
3190	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3191	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3192	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3193	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3194	\|\| (pCtx->ds.Attr.n.u1Granularity));
3195	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3196	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3197	}
3198	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3199	{
3200	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3201	Assert(pCtx->es.Attr.n.u1Present);
3202	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3203	Assert(!(pCtx->es.Attr.u & 0xf00));
3204	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3205	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3206	\|\| !(pCtx->es.Attr.n.u1Granularity));
3207	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3208	\|\| (pCtx->es.Attr.n.u1Granularity));
3209	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3210	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3211	}
3212	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3213	{
3214	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3215	Assert(pCtx->fs.Attr.n.u1Present);
3216	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3217	Assert(!(pCtx->fs.Attr.u & 0xf00));
3218	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3219	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3220	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3221	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3222	\|\| (pCtx->fs.Attr.n.u1Granularity));
3223	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3224	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3225	}
3226	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3227	{
3228	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3229	Assert(pCtx->gs.Attr.n.u1Present);
3230	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3231	Assert(!(pCtx->gs.Attr.u & 0xf00));
3232	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3233	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3234	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3235	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3236	\|\| (pCtx->gs.Attr.n.u1Granularity));
3237	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3238	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3239	}
3240	/* 64-bit capable CPUs. */
3241	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3242	Assert(!(pCtx->cs.u64Base >> 32));
3243	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3244	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3245	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3246	# endif
3247	}
3248	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3249	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3250	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3251	{
3252	/* Real and v86 mode checks. */
3253	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3254	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3255	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3256	{
3257	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3258	}
3259	else
3260	{
3261	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3262	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3263	}
3264
3265	/* CS */
3266	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3267	Assert(pCtx->cs.u32Limit == 0xffff);
3268	Assert(u32CSAttr == 0xf3);
3269	/* SS */
3270	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3271	Assert(pCtx->ss.u32Limit == 0xffff);
3272	Assert(u32SSAttr == 0xf3);
3273	/* DS */
3274	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3275	Assert(pCtx->ds.u32Limit == 0xffff);
3276	Assert(u32DSAttr == 0xf3);
3277	/* ES */
3278	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3279	Assert(pCtx->es.u32Limit == 0xffff);
3280	Assert(u32ESAttr == 0xf3);
3281	/* FS */
3282	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3283	Assert(pCtx->fs.u32Limit == 0xffff);
3284	Assert(u32FSAttr == 0xf3);
3285	/* GS */
3286	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3287	Assert(pCtx->gs.u32Limit == 0xffff);
3288	Assert(u32GSAttr == 0xf3);
3289	/* 64-bit capable CPUs. */
3290	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3291	Assert(!(pCtx->cs.u64Base >> 32));
3292	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3293	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3294	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3295	# endif
3296	}
3297	}
3298	#endif /* VBOX_STRICT */
3299
3300
3301	/**
3302	* Writes a guest segment register into the guest-state area in the VMCS.
3303	*
3304	* @returns VBox status code.
3305	* @param pVCpu Pointer to the VMCPU.
3306	* @param idxSel Index of the selector in the VMCS.
3307	* @param idxLimit Index of the segment limit in the VMCS.
3308	* @param idxBase Index of the segment base in the VMCS.
3309	* @param idxAccess Index of the access rights of the segment in the VMCS.
3310	* @param pSelReg Pointer to the segment selector.
3311	* @param pCtx Pointer to the guest-CPU context.
3312	*
3313	* @remarks No-long-jump zone!!!
3314	*/
3315	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3316	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3317	{
3318	int rc;
3319	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3320	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3321	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3322	AssertRCReturn(rc, rc);
3323
3324	uint32_t u32Access = pSelReg->Attr.u;
3325	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3326	{
3327	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3328	u32Access = 0xf3;
3329	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3330	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3331	}
3332	else
3333	{
3334	/*
3335	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3336	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3337	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3338	* loaded in protected-mode have their attribute as 0.
3339	*/
3340	if (!u32Access)
3341	u32Access = HMVMX_SEL_UNUSABLE;
3342	}
3343
3344	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3345	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3346	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3347
3348	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3349	AssertRCReturn(rc, rc);
3350	return rc;
3351	}
3352
3353
3354	/**
3355	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3356	* into the guest-state area in the VMCS.
3357	*
3358	* @returns VBox status code.
3359	* @param pVM Pointer to the VM.
3360	* @param pVCPU Pointer to the VMCPU.
3361	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3362	* out-of-sync. Make sure to update the required fields
3363	* before using them.
3364	*
3365	* @remarks No-long-jump zone!!!
3366	*/
3367	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3368	{
3369	int rc = VERR_INTERNAL_ERROR_5;
3370	PVM pVM = pVCpu->CTX_SUFF(pVM);
3371
3372	/*
3373	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3374	*/
3375	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3376	{
3377	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3378	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3379	{
3380	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3381	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3382	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3383	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3384	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3385	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3386	}
3387
3388	#ifdef VBOX_WITH_REM
3389	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3390	{
3391	Assert(pVM->hm.s.vmx.pRealModeTSS);
3392	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3393	if ( pVCpu->hm.s.vmx.fWasInRealMode
3394	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3395	{
3396	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3397	in real-mode (e.g. OpenBSD 4.0) */
3398	REMFlushTBs(pVM);
3399	Log(("Load: Switch to protected mode detected!\n"));
3400	pVCpu->hm.s.vmx.fWasInRealMode = false;
3401	}
3402	}
3403	#endif
3404	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3405	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3406	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3407	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3408	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3409	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3410	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3411	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3412	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3413	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3414	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3415	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3416	AssertRCReturn(rc, rc);
3417
3418	#ifdef VBOX_STRICT
3419	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3420	#endif
3421	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3422	}
3423
3424	/*
3425	* Guest TR.
3426	*/
3427	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3428	{
3429	/*
3430	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3431	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3432	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3433	*/
3434	uint16_t u16Sel = 0;
3435	uint32_t u32Limit = 0;
3436	uint64_t u64Base = 0;
3437	uint32_t u32AccessRights = 0;
3438
3439	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3440	{
3441	u16Sel = pMixedCtx->tr.Sel;
3442	u32Limit = pMixedCtx->tr.u32Limit;
3443	u64Base = pMixedCtx->tr.u64Base;
3444	u32AccessRights = pMixedCtx->tr.Attr.u;
3445	}
3446	else
3447	{
3448	Assert(pVM->hm.s.vmx.pRealModeTSS);
3449	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3450
3451	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3452	RTGCPHYS GCPhys;
3453	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3454	AssertRCReturn(rc, rc);
3455
3456	X86DESCATTR DescAttr;
3457	DescAttr.u = 0;
3458	DescAttr.n.u1Present = 1;
3459	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3460
3461	u16Sel = 0;
3462	u32Limit = HM_VTX_TSS_SIZE;
3463	u64Base = GCPhys; /* in real-mode phys = virt. */
3464	u32AccessRights = DescAttr.u;
3465	}
3466
3467	/* Validate. */
3468	Assert(!(u16Sel & RT_BIT(2)));
3469	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3470	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3471	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3472	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3473	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3474	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3475	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3476	Assert( (u32Limit & 0xfff) == 0xfff
3477	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3478	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3479	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3480
3481	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3482	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3483	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3484	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3485	AssertRCReturn(rc, rc);
3486
3487	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3488	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3489	}
3490
3491	/*
3492	* Guest GDTR.
3493	*/
3494	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3495	{
3496	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3497	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3498	AssertRCReturn(rc, rc);
3499
3500	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3501	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3502	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3503	}
3504
3505	/*
3506	* Guest LDTR.
3507	*/
3508	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3509	{
3510	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3511	uint32_t u32Access = 0;
3512	if (!pMixedCtx->ldtr.Attr.u)
3513	u32Access = HMVMX_SEL_UNUSABLE;
3514	else
3515	u32Access = pMixedCtx->ldtr.Attr.u;
3516
3517	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3518	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3519	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3520	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3521	AssertRCReturn(rc, rc);
3522
3523	/* Validate. */
3524	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3525	{
3526	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3527	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3528	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3529	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3530	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3531	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3532	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3533	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3534	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3535	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3536	}
3537
3538	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3539	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3540	}
3541
3542	/*
3543	* Guest IDTR.
3544	*/
3545	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3546	{
3547	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3548	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3549	AssertRCReturn(rc, rc);
3550
3551	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3552	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3553	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3554	}
3555
3556	return VINF_SUCCESS;
3557	}
3558
3559
3560	/**
3561	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3562	* areas. These MSRs will automatically be loaded to the host CPU on every
3563	* successful VM entry and stored from the host CPU on every successful VM exit.
3564	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3565	*
3566	* @returns VBox status code.
3567	* @param pVCpu Pointer to the VMCPU.
3568	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3569	* out-of-sync. Make sure to update the required fields
3570	* before using them.
3571	*
3572	* @remarks No-long-jump zone!!!
3573	*/
3574	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3575	{
3576	AssertPtr(pVCpu);
3577	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3578
3579	/*
3580	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3581	*/
3582	int rc = VINF_SUCCESS;
3583	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3584	{
3585	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3586	PVM pVM = pVCpu->CTX_SUFF(pVM);
3587	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3588	uint32_t cGuestMsrs = 0;
3589
3590	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3591	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3592	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3593	if (fSupportsNX \|\| fSupportsLongMode)
3594	{
3595	/** @todo support save IA32_EFER, i.e.
3596	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3597	* guest EFER need not be part of the VM-entry MSR-load area. */
3598	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3599	pGuestMsr->u32Reserved = 0;
3600	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3601	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3602	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3603	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3604	pGuestMsr++; cGuestMsrs++;
3605	if (fSupportsLongMode)
3606	{
3607	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3608	pGuestMsr->u32Reserved = 0;
3609	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3610	pGuestMsr++; cGuestMsrs++;
3611	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3612	pGuestMsr->u32Reserved = 0;
3613	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3614	pGuestMsr++; cGuestMsrs++;
3615	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3618	pGuestMsr++; cGuestMsrs++;
3619	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3620	pGuestMsr->u32Reserved = 0;
3621	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3622	pGuestMsr++; cGuestMsrs++;
3623	}
3624	}
3625
3626	/*
3627	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3628	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3629	*/
3630	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3631	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3632	{
3633	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3634	pGuestMsr->u32Reserved = 0;
3635	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3636	AssertRCReturn(rc, rc);
3637	pGuestMsr++; cGuestMsrs++;
3638	}
3639
3640	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3641	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3642	{
3643	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3644	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3645	}
3646
3647	/* Update the VCPU's copy of the guest MSR count. */
3648	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3649	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3650	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3651	AssertRCReturn(rc, rc);
3652	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3653
3654	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3655	}
3656
3657	/*
3658	* Guest Sysenter MSRs.
3659	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3660	* VM-exits on WRMSRs for these MSRs.
3661	*/
3662	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3663	{
3664	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3665	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3666	}
3667	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3668	{
3669	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3670	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3671	}
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3673	{
3674	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3675	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3676	}
3677	AssertRCReturn(rc, rc);
3678
3679	return rc;
3680	}
3681
3682
3683	/**
3684	* Loads the guest activity state into the guest-state area in the VMCS.
3685	*
3686	* @returns VBox status code.
3687	* @param pVCpu Pointer to the VMCPU.
3688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3689	* out-of-sync. Make sure to update the required fields
3690	* before using them.
3691	*
3692	* @remarks No-long-jump zone!!!
3693	*/
3694	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3695	{
3696	/** @todo See if we can make use of other states, e.g.
3697	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3698	int rc = VINF_SUCCESS;
3699	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3700	{
3701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3702	AssertRCReturn(rc, rc);
3703	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3704	}
3705	return rc;
3706	}
3707
3708
3709	/**
3710	* Sets up the appropriate function to run guest code.
3711	*
3712	* @returns VBox status code.
3713	* @param pVCpu Pointer to the VMCPU.
3714	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3715	* out-of-sync. Make sure to update the required fields
3716	* before using them.
3717	*
3718	* @remarks No-long-jump zone!!!
3719	*/
3720	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3721	{
3722	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3723	{
3724	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3725	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3726	#endif
3727	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3728	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3729	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3730	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3731	#else
3732	/* 64-bit host or hybrid host. */
3733	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3734	#endif
3735	}
3736	else
3737	{
3738	/* Guest is not in long mode, use the 32-bit handler. */
3739	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3740	}
3741	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3742	return VINF_SUCCESS;
3743	}
3744
3745
3746	/**
3747	* Wrapper for running the guest code in VT-x.
3748	*
3749	* @returns VBox strict status code.
3750	* @param pVM Pointer to the VM.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param pCtx Pointer to the guest-CPU context.
3753	*
3754	* @remarks No-long-jump zone!!!
3755	*/
3756	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3757	{
3758	/*
3759	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3760	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3761	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3762	*/
3763	#ifdef VBOX_WITH_KERNEL_USING_XMM
3764	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3765	#else
3766	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3767	#endif
3768	}
3769
3770
3771	/**
3772	* Report world-switch error and dump some useful debug info.
3773	*
3774	* @param pVM Pointer to the VM.
3775	* @param pVCpu Pointer to the VMCPU.
3776	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3777	* @param pCtx Pointer to the guest-CPU context.
3778	* @param pVmxTransient Pointer to the VMX transient structure (only
3779	* exitReason updated).
3780	*/
3781	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3782	{
3783	Assert(pVM);
3784	Assert(pVCpu);
3785	Assert(pCtx);
3786	Assert(pVmxTransient);
3787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3788
3789	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3790	switch (rcVMRun)
3791	{
3792	case VERR_VMX_INVALID_VMXON_PTR:
3793	AssertFailed();
3794	break;
3795	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3796	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3797	{
3798	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3799	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3800	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3801	AssertRC(rc);
3802
3803	#ifdef VBOX_STRICT
3804	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3805	pVmxTransient->uExitReason));
3806	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3807	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3808	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3809	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3810	else
3811	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3812
3813	/* VMX control bits. */
3814	uint32_t u32Val;
3815	uint64_t u64Val;
3816	HMVMXHCUINTREG uHCReg;
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3845	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3846	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3847	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3848	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3849	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3850	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3852	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3853	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3854	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3855	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3856	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3857	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3858	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3859
3860	/* Guest bits. */
3861	RTGCUINTREG uGCReg;
3862	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3863	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3864	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3865	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3866	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3867	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3868	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3869	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3870
3871	/* Host bits. */
3872	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3873	Log(("Host CR0 %#RHr\n", uHCReg));
3874	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3875	Log(("Host CR3 %#RHr\n", uHCReg));
3876	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3877	Log(("Host CR4 %#RHr\n", uHCReg));
3878
3879	RTGDTR HostGdtr;
3880	PCX86DESCHC pDesc;
3881	ASMGetGDTR(&HostGdtr);
3882	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3883	Log(("Host CS %#08x\n", u32Val));
3884	if (u32Val < HostGdtr.cbGdt)
3885	{
3886	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3887	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3888	}
3889
3890	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3891	Log(("Host DS %#08x\n", u32Val));
3892	if (u32Val < HostGdtr.cbGdt)
3893	{
3894	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3895	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3896	}
3897
3898	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3899	Log(("Host ES %#08x\n", u32Val));
3900	if (u32Val < HostGdtr.cbGdt)
3901	{
3902	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3903	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3904	}
3905
3906	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3907	Log(("Host FS %#08x\n", u32Val));
3908	if (u32Val < HostGdtr.cbGdt)
3909	{
3910	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3911	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3912	}
3913
3914	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3915	Log(("Host GS %#08x\n", u32Val));
3916	if (u32Val < HostGdtr.cbGdt)
3917	{
3918	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3919	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3920	}
3921
3922	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3923	Log(("Host SS %#08x\n", u32Val));
3924	if (u32Val < HostGdtr.cbGdt)
3925	{
3926	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3927	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3928	}
3929
3930	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3931	Log(("Host TR %#08x\n", u32Val));
3932	if (u32Val < HostGdtr.cbGdt)
3933	{
3934	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3935	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3936	}
3937
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3939	Log(("Host TR Base %#RHv\n", uHCReg));
3940	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3941	Log(("Host GDTR Base %#RHv\n", uHCReg));
3942	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3943	Log(("Host IDTR Base %#RHv\n", uHCReg));
3944	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3945	Log(("Host SYSENTER CS %#08x\n", u32Val));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3947	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3949	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3950	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3951	Log(("Host RSP %#RHv\n", uHCReg));
3952	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3953	Log(("Host RIP %#RHv\n", uHCReg));
3954	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3955	if (HMVMX_IS_64BIT_HOST_MODE())
3956	{
3957	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3958	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3959	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3960	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3961	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3962	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3963	}
3964	# endif
3965	#endif /* VBOX_STRICT */
3966	break;
3967	}
3968
3969	default:
3970	/* Impossible */
3971	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3972	break;
3973	}
3974	NOREF(pVM);
3975	}
3976
3977
3978	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3979	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3980	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3981	#endif
3982	#ifdef VBOX_STRICT
3983	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3984	{
3985	switch (idxField)
3986	{
3987	case VMX_VMCS_GUEST_RIP:
3988	case VMX_VMCS_GUEST_RSP:
3989	case VMX_VMCS_GUEST_DR7:
3990	case VMX_VMCS_GUEST_SYSENTER_EIP:
3991	case VMX_VMCS_GUEST_SYSENTER_ESP:
3992	case VMX_VMCS_GUEST_GDTR_BASE:
3993	case VMX_VMCS_GUEST_IDTR_BASE:
3994	case VMX_VMCS_GUEST_CS_BASE:
3995	case VMX_VMCS_GUEST_DS_BASE:
3996	case VMX_VMCS_GUEST_ES_BASE:
3997	case VMX_VMCS_GUEST_FS_BASE:
3998	case VMX_VMCS_GUEST_GS_BASE:
3999	case VMX_VMCS_GUEST_SS_BASE:
4000	case VMX_VMCS_GUEST_LDTR_BASE:
4001	case VMX_VMCS_GUEST_TR_BASE:
4002	case VMX_VMCS_GUEST_CR3:
4003	return true;
4004	}
4005	return false;
4006	}
4007
4008	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4009	{
4010	switch (idxField)
4011	{
4012	/* Read-only fields. */
4013	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4014	return true;
4015	}
4016	/* All readable fields should also be part of the VMCS write cache. */
4017	return hmR0VmxIsValidWriteField(idxField);
4018	}
4019	#endif /* VBOX_STRICT */
4020
4021	/**
4022	* Executes the specified handler in 64-bit mode.
4023	*
4024	* @returns VBox status code.
4025	* @param pVM Pointer to the VM.
4026	* @param pVCpu Pointer to the VMCPU.
4027	* @param pCtx Pointer to the guest CPU context.
4028	* @param enmOp The operation to perform.
4029	* @param cbParam Number of parameters.
4030	* @param paParam Array of 32-bit parameters.
4031	*/
4032	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4033	uint32_t *paParam)
4034	{
4035	int rc, rc2;
4036	PHMGLOBLCPUINFO pCpu;
4037	RTHCPHYS HCPhysCpuPage;
4038	RTCCUINTREG uOldEFlags;
4039
4040	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4041	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4042	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4043	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4044
4045	#ifdef VBOX_STRICT
4046	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4047	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4048
4049	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4050	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4051	#endif
4052
4053	/* Disable interrupts. */
4054	uOldEFlags = ASMIntDisableFlags();
4055
4056	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4057	RTCPUID idHostCpu = RTMpCpuId();
4058	CPUMR0SetLApic(pVM, idHostCpu);
4059	#endif
4060
4061	pCpu = HMR0GetCurrentCpu();
4062	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4063
4064	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4065	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4066
4067	/* Leave VMX Root Mode. */
4068	VMXDisable();
4069
4070	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4071
4072	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4073	CPUMSetHyperEIP(pVCpu, enmOp);
4074	for (int i = (int)cbParam - 1; i >= 0; i--)
4075	CPUMPushHyper(pVCpu, paParam[i]);
4076
4077	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4078
4079	/* Call the switcher. */
4080	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4081	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4082
4083	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4084	/* Make sure the VMX instructions don't cause #UD faults. */
4085	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4086
4087	/* Re-enter VMX Root Mode */
4088	rc2 = VMXEnable(HCPhysCpuPage);
4089	if (RT_FAILURE(rc2))
4090	{
4091	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4092	ASMSetFlags(uOldEFlags);
4093	return rc2;
4094	}
4095
4096	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4097	AssertRC(rc2);
4098	Assert(!(ASMGetFlags() & X86_EFL_IF));
4099	ASMSetFlags(uOldEFlags);
4100	return rc;
4101	}
4102
4103
4104	/**
4105	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4106	* supporting 64-bit guests.
4107	*
4108	* @returns VBox status code.
4109	* @param fResume Whether to VMLAUNCH or VMRESUME.
4110	* @param pCtx Pointer to the guest-CPU context.
4111	* @param pCache Pointer to the VMCS cache.
4112	* @param pVM Pointer to the VM.
4113	* @param pVCpu Pointer to the VMCPU.
4114	*/
4115	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4116	{
4117	uint32_t aParam[6];
4118	PHMGLOBLCPUINFO pCpu = NULL;
4119	RTHCPHYS HCPhysCpuPage = 0;
4120	int rc = VERR_INTERNAL_ERROR_5;
4121	AssertReturn(pVM->hm.s.pfnVMXGCStartVM64, VERR_HM_IPE_5);
4122
4123	pCpu = HMR0GetCurrentCpu();
4124	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4125
4126	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4127	pCache->uPos = 1;
4128	pCache->interPD = PGMGetInterPaeCR3(pVM);
4129	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4130	#endif
4131
4132	#ifdef VBOX_STRICT
4133	pCache->TestIn.HCPhysCpuPage = 0;
4134	pCache->TestIn.HCPhysVmcs = 0;
4135	pCache->TestIn.pCache = 0;
4136	pCache->TestOut.HCPhysVmcs = 0;
4137	pCache->TestOut.pCache = 0;
4138	pCache->TestOut.pCtx = 0;
4139	pCache->TestOut.eflags = 0;
4140	#endif
4141
4142	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4143	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4144	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4145	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4146	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4147	aParam[5] = 0;
4148
4149	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4150	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4151	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4152	#endif
4153	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4154
4155	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4156	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4157	Assert(pCtx->dr[4] == 10);
4158	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4159	#endif
4160
4161	#ifdef VBOX_STRICT
4162	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4163	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4164	pVCpu->hm.s.vmx.HCPhysVmcs));
4165	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4166	pCache->TestOut.HCPhysVmcs));
4167	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4168	pCache->TestOut.pCache));
4169	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4170	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4171	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4172	pCache->TestOut.pCtx));
4173	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4174	#endif
4175	return rc;
4176	}
4177
4178
4179	/**
4180	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4181	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4182	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4183	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4184	*
4185	* @returns VBox status code.
4186	* @param pVM Pointer to the VM.
4187	* @param pVCpu Pointer to the VMCPU.
4188	*/
4189	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4190	{
4191	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4192	{ \
4193	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4194	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4195	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4196	++cReadFields; \
4197	}
4198
4199	AssertPtr(pVM);
4200	AssertPtr(pVCpu);
4201	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4202	uint32_t cReadFields = 0;
4203
4204	/* Guest-natural selector base fields */
4205	#if 0
4206	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4208	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4209	#endif
4210	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4218	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4219	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4220	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4221	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4222	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4223	#if 0
4224	/* Unused natural width guest-state fields. */
4225	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4226	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4227	#endif
4228	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4230
4231	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4232	#if 0
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4242	#endif
4243
4244	/* Natural width guest-state fields. */
4245	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4246	#if 0
4247	/* Currently unused field. */
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4249	#endif
4250
4251	if (pVM->hm.s.fNestedPaging)
4252	{
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4254	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4255	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4256	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4257	}
4258	else
4259	{
4260	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4261	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4262	}
4263
4264	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4265	return VINF_SUCCESS;
4266	}
4267
4268
4269	/**
4270	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4271	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4272	* darwin, running 64-bit guests).
4273	*
4274	* @returns VBox status code.
4275	* @param pVCpu Pointer to the VMCPU.
4276	* @param idxField The VMCS field encoding.
4277	* @param u64Val 16, 32 or 64 bits value.
4278	*/
4279	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4280	{
4281	int rc;
4282	switch (idxField)
4283	{
4284	/*
4285	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4286	*/
4287	/* 64-bit Control fields. */
4288	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4289	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4290	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4291	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4292	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4293	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4294	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4295	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4296	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4297	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4298	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4299	case VMX_VMCS64_CTRL_EPTP_FULL:
4300	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4301	/* 64-bit Guest-state fields. */
4302	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4303	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4304	case VMX_VMCS64_GUEST_PAT_FULL:
4305	case VMX_VMCS64_GUEST_EFER_FULL:
4306	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4307	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4308	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4309	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4310	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4311	/* 64-bit Host-state fields. */
4312	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4313	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4314	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4315	{
4316	rc = VMXWriteVmcs32(idxField, u64Val);
4317	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4318	break;
4319	}
4320
4321	/*
4322	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4323	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4324	*/
4325	/* Natural-width Guest-state fields. */
4326	case VMX_VMCS_GUEST_CR3:
4327	case VMX_VMCS_GUEST_ES_BASE:
4328	case VMX_VMCS_GUEST_CS_BASE:
4329	case VMX_VMCS_GUEST_SS_BASE:
4330	case VMX_VMCS_GUEST_DS_BASE:
4331	case VMX_VMCS_GUEST_FS_BASE:
4332	case VMX_VMCS_GUEST_GS_BASE:
4333	case VMX_VMCS_GUEST_LDTR_BASE:
4334	case VMX_VMCS_GUEST_TR_BASE:
4335	case VMX_VMCS_GUEST_GDTR_BASE:
4336	case VMX_VMCS_GUEST_IDTR_BASE:
4337	case VMX_VMCS_GUEST_DR7:
4338	case VMX_VMCS_GUEST_RSP:
4339	case VMX_VMCS_GUEST_RIP:
4340	case VMX_VMCS_GUEST_SYSENTER_ESP:
4341	case VMX_VMCS_GUEST_SYSENTER_EIP:
4342	{
4343	if (!(u64Val >> 32))
4344	{
4345	/* If this field is 64-bit, VT-x will zero out the top bits. */
4346	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4347	}
4348	else
4349	{
4350	/* Assert that only the 32->64 switcher case should ever come here. */
4351	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4352	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4353	}
4354	break;
4355	}
4356
4357	default:
4358	{
4359	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4360	rc = VERR_INVALID_PARAMETER;
4361	break;
4362	}
4363	}
4364	AssertRCReturn(rc, rc);
4365	return rc;
4366	}
4367
4368
4369	/**
4370	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4371	* hosts (except darwin) for 64-bit guests.
4372	*
4373	* @param pVCpu Pointer to the VMCPU.
4374	* @param idxField The VMCS field encoding.
4375	* @param u64Val 16, 32 or 64 bits value.
4376	*/
4377	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4378	{
4379	AssertPtr(pVCpu);
4380	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4381
4382	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4383	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4384
4385	/* Make sure there are no duplicates. */
4386	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4387	{
4388	if (pCache->Write.aField[i] == idxField)
4389	{
4390	pCache->Write.aFieldVal[i] = u64Val;
4391	return VINF_SUCCESS;
4392	}
4393	}
4394
4395	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4396	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4397	pCache->Write.cValidEntries++;
4398	return VINF_SUCCESS;
4399	}
4400
4401	/* Enable later when the assembly code uses these as callbacks. */
4402	#if 0
4403	/*
4404	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4405	*
4406	* @param pVCpu Pointer to the VMCPU.
4407	* @param pCache Pointer to the VMCS cache.
4408	*
4409	* @remarks No-long-jump zone!!!
4410	*/
4411	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4412	{
4413	AssertPtr(pCache);
4414	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4415	{
4416	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4417	AssertRC(rc);
4418	}
4419	pCache->Write.cValidEntries = 0;
4420	}
4421
4422
4423	/**
4424	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4425	*
4426	* @param pVCpu Pointer to the VMCPU.
4427	* @param pCache Pointer to the VMCS cache.
4428	*
4429	* @remarks No-long-jump zone!!!
4430	*/
4431	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4432	{
4433	AssertPtr(pCache);
4434	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4435	{
4436	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4437	AssertRC(rc);
4438	}
4439	}
4440	#endif
4441	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4442
4443
4444	/**
4445	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4446	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4447	* timer.
4448	*
4449	* @returns VBox status code.
4450	* @param pVCpu Pointer to the VMCPU.
4451	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4452	* out-of-sync. Make sure to update the required fields
4453	* before using them.
4454	* @remarks No-long-jump zone!!!
4455	*/
4456	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4457	{
4458	int rc = VERR_INTERNAL_ERROR_5;
4459	bool fOffsettedTsc = false;
4460	PVM pVM = pVCpu->CTX_SUFF(pVM);
4461	if (pVM->hm.s.vmx.fUsePreemptTimer)
4462	{
4463	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4464
4465	/* Make sure the returned values have sane upper and lower boundaries. */
4466	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4467	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4468	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4469	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4470
4471	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4472	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4473	}
4474	else
4475	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4476
4477	if (fOffsettedTsc)
4478	{
4479	uint64_t u64CurTSC = ASMReadTSC();
4480	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4481	{
4482	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4483	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4484
4485	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4486	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4487	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4488	}
4489	else
4490	{
4491	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4492	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4493	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4494	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4495	}
4496	}
4497	else
4498	{
4499	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4500	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4501	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4502	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4503	}
4504	}
4505
4506
4507	/**
4508	* Determines if an exception is a contributory exception. Contributory
4509	* exceptions are ones which can cause double-faults. Page-fault is
4510	* intentionally not included here as it's a conditional contributory exception.
4511	*
4512	* @returns true if the exception is contributory, false otherwise.
4513	* @param uVector The exception vector.
4514	*/
4515	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4516	{
4517	switch (uVector)
4518	{
4519	case X86_XCPT_GP:
4520	case X86_XCPT_SS:
4521	case X86_XCPT_NP:
4522	case X86_XCPT_TS:
4523	case X86_XCPT_DE:
4524	return true;
4525	default:
4526	break;
4527	}
4528	return false;
4529	}
4530
4531
4532	/**
4533	* Sets an event as a pending event to be injected into the guest.
4534	*
4535	* @param pVCpu Pointer to the VMCPU.
4536	* @param u32IntrInfo The VM-entry interruption-information field.
4537	* @param cbInstr The VM-entry instruction length in bytes (for software
4538	* interrupts, exceptions and privileged software
4539	* exceptions).
4540	* @param u32ErrCode The VM-entry exception error code.
4541	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4542	* page-fault.
4543	*/
4544	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4545	RTGCUINTPTR GCPtrFaultAddress)
4546	{
4547	Assert(!pVCpu->hm.s.Event.fPending);
4548	pVCpu->hm.s.Event.fPending = true;
4549	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4550	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4551	pVCpu->hm.s.Event.cbInstr = cbInstr;
4552	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4553	}
4554
4555
4556	/**
4557	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4558	*
4559	* @param pVCpu Pointer to the VMCPU.
4560	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4561	* out-of-sync. Make sure to update the required fields
4562	* before using them.
4563	*/
4564	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4565	{
4566	/* Inject the double-fault. */
4567	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4568	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4569	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4570	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4571	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4572	}
4573
4574
4575	/**
4576	* Handle a condition that occurred while delivering an event through the guest
4577	* IDT.
4578	*
4579	* @returns VBox status code (informational error codes included).
4580	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4581	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4582	* continue execution of the guest which will delivery the #DF.
4583	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4584	*
4585	* @param pVCpu Pointer to the VMCPU.
4586	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4587	* out-of-sync. Make sure to update the required fields
4588	* before using them.
4589	* @param pVmxTransient Pointer to the VMX transient structure.
4590	*
4591	* @remarks No-long-jump zone!!!
4592	*/
4593	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4594	{
4595	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4596	AssertRC(rc);
4597	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4598	{
4599	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4600	AssertRCReturn(rc, rc);
4601
4602	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4603	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4604	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4605
4606	typedef enum
4607	{
4608	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4609	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4610	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4611	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4612	} VMXREFLECTXCPT;
4613
4614	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4615	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4616	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4617	{
4618	enmReflect = VMXREFLECTXCPT_XCPT;
4619	#ifdef VBOX_STRICT
4620	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4621	&& uExitVector == X86_XCPT_PF)
4622	{
4623	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4624	}
4625	#endif
4626	if ( uExitVector == X86_XCPT_PF
4627	&& uIdtVector == X86_XCPT_PF)
4628	{
4629	pVmxTransient->fVectoringPF = true;
4630	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4631	}
4632	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4633	&& hmR0VmxIsContributoryXcpt(uExitVector)
4634	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4635	\|\| uIdtVector == X86_XCPT_PF))
4636	{
4637	enmReflect = VMXREFLECTXCPT_DF;
4638	}
4639	else if (uIdtVector == X86_XCPT_DF)
4640	enmReflect = VMXREFLECTXCPT_TF;
4641	}
4642	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4643	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4644	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4645	{
4646	/*
4647	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4648	* (whatever they are) as they reoccur when restarting the instruction.
4649	*/
4650	enmReflect = VMXREFLECTXCPT_XCPT;
4651	}
4652
4653	switch (enmReflect)
4654	{
4655	case VMXREFLECTXCPT_XCPT:
4656	{
4657	uint32_t u32ErrCode = 0;
4658	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4659	{
4660	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4661	AssertRCReturn(rc, rc);
4662	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4663	}
4664
4665	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4666	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4667	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4668	rc = VINF_SUCCESS;
4669	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4670	break;
4671	}
4672
4673	case VMXREFLECTXCPT_DF:
4674	{
4675	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4676	rc = VINF_VMX_DOUBLE_FAULT;
4677	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4678	uExitVector));
4679	break;
4680	}
4681
4682	case VMXREFLECTXCPT_TF:
4683	{
4684	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4685	rc = VINF_EM_RESET;
4686	break;
4687	}
4688
4689	default:
4690	Assert(rc == VINF_SUCCESS);
4691	break;
4692	}
4693	}
4694	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4695	return rc;
4696	}
4697
4698
4699	/**
4700	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4701	*
4702	* @returns VBox status code.
4703	* @param pVCpu Pointer to the VMCPU.
4704	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4705	* out-of-sync. Make sure to update the required fields
4706	* before using them.
4707	*
4708	* @remarks No-long-jump zone!!!
4709	*/
4710	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4711	{
4712	int rc = VINF_SUCCESS;
4713	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4714	{
4715	uint32_t uVal = 0;
4716	uint32_t uShadow = 0;
4717	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4718	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4719	AssertRCReturn(rc, rc);
4720	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4721	CPUMSetGuestCR0(pVCpu, uVal);
4722	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4723	}
4724	return rc;
4725	}
4726
4727
4728	/**
4729	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4730	*
4731	* @returns VBox status code.
4732	* @param pVCpu Pointer to the VMCPU.
4733	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4734	* out-of-sync. Make sure to update the required fields
4735	* before using them.
4736	*
4737	* @remarks No-long-jump zone!!!
4738	*/
4739	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4740	{
4741	int rc = VINF_SUCCESS;
4742	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4743	{
4744	uint32_t uVal = 0;
4745	uint32_t uShadow = 0;
4746	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4747	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4748	AssertRCReturn(rc, rc);
4749	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4750	CPUMSetGuestCR4(pVCpu, uVal);
4751	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4752	}
4753	return rc;
4754	}
4755
4756
4757	/**
4758	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4759	*
4760	* @returns VBox status code.
4761	* @param pVCpu Pointer to the VMCPU.
4762	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4763	* out-of-sync. Make sure to update the required fields
4764	* before using them.
4765	*
4766	* @remarks No-long-jump zone!!!
4767	*/
4768	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4769	{
4770	int rc = VINF_SUCCESS;
4771	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4772	{
4773	RTGCUINTREG uVal = 0;
4774	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4775	AssertRCReturn(rc, rc);
4776	pMixedCtx->rip = uVal;
4777	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4778	}
4779	return rc;
4780	}
4781
4782
4783	/**
4784	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4785	*
4786	* @returns VBox status code.
4787	* @param pVCpu Pointer to the VMCPU.
4788	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4789	* out-of-sync. Make sure to update the required fields
4790	* before using them.
4791	*
4792	* @remarks No-long-jump zone!!!
4793	*/
4794	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4795	{
4796	int rc = VINF_SUCCESS;
4797	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4798	{
4799	RTGCUINTREG uVal = 0;
4800	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4801	AssertRCReturn(rc, rc);
4802	pMixedCtx->rsp = uVal;
4803	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4804	}
4805	return rc;
4806	}
4807
4808
4809	/**
4810	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4811	*
4812	* @returns VBox status code.
4813	* @param pVCpu Pointer to the VMCPU.
4814	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4815	* out-of-sync. Make sure to update the required fields
4816	* before using them.
4817	*
4818	* @remarks No-long-jump zone!!!
4819	*/
4820	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4821	{
4822	int rc = VINF_SUCCESS;
4823	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4824	{
4825	uint32_t uVal = 0;
4826	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4827	AssertRCReturn(rc, rc);
4828	pMixedCtx->eflags.u32 = uVal;
4829
4830	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4831	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4832	{
4833	PVM pVM = pVCpu->CTX_SUFF(pVM);
4834	Assert(pVM->hm.s.vmx.pRealModeTSS);
4835	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4836	pMixedCtx->eflags.Bits.u1VM = 0;
4837	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4838	}
4839
4840	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4841	}
4842	return rc;
4843	}
4844
4845
4846	/**
4847	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4848	* guest-CPU context.
4849	*/
4850	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4851	{
4852	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4853	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4854	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4855	return rc;
4856	}
4857
4858
4859	/**
4860	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4861	* from the guest-state area in the VMCS.
4862	*
4863	* @param pVCpu Pointer to the VMCPU.
4864	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4865	* out-of-sync. Make sure to update the required fields
4866	* before using them.
4867	*
4868	* @remarks No-long-jump zone!!!
4869	*/
4870	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4871	{
4872	uint32_t uIntrState = 0;
4873	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4874	AssertRC(rc);
4875
4876	if (!uIntrState)
4877	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4878	else
4879	{
4880	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4881	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4882	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4883	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4884	AssertRC(rc);
4885	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4886	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4887	}
4888	}
4889
4890
4891	/**
4892	* Saves the guest's activity state.
4893	*
4894	* @returns VBox status code.
4895	* @param pVCpu Pointer to the VMCPU.
4896	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4897	* out-of-sync. Make sure to update the required fields
4898	* before using them.
4899	*
4900	* @remarks No-long-jump zone!!!
4901	*/
4902	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4903	{
4904	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4905	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4906	return VINF_SUCCESS;
4907	}
4908
4909
4910	/**
4911	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4912	* the current VMCS into the guest-CPU context.
4913	*
4914	* @returns VBox status code.
4915	* @param pVCpu Pointer to the VMCPU.
4916	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4917	* out-of-sync. Make sure to update the required fields
4918	* before using them.
4919	*
4920	* @remarks No-long-jump zone!!!
4921	*/
4922	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4923	{
4924	int rc = VINF_SUCCESS;
4925	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4926	{
4927	uint32_t u32Val = 0;
4928	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4929	pMixedCtx->SysEnter.cs = u32Val;
4930	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4931	}
4932
4933	RTGCUINTREG uGCVal = 0;
4934	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4935	{
4936	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4937	pMixedCtx->SysEnter.eip = uGCVal;
4938	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4939	}
4940	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4941	{
4942	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4943	pMixedCtx->SysEnter.esp = uGCVal;
4944	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4945	}
4946	return rc;
4947	}
4948
4949
4950	/**
4951	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4952	* context.
4953	*
4954	* @returns VBox status code.
4955	* @param pVCpu Pointer to the VMCPU.
4956	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4957	* out-of-sync. Make sure to update the required fields
4958	* before using them.
4959	*
4960	* @remarks No-long-jump zone!!!
4961	*/
4962	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4963	{
4964	int rc = VINF_SUCCESS;
4965	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4966	{
4967	RTGCUINTREG uVal = 0;
4968	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4969	pMixedCtx->fs.u64Base = uVal;
4970	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4971	}
4972	return rc;
4973	}
4974
4975
4976	/**
4977	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4978	* context.
4979	*
4980	* @returns VBox status code.
4981	* @param pVCpu Pointer to the VMCPU.
4982	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4983	* out-of-sync. Make sure to update the required fields
4984	* before using them.
4985	*
4986	* @remarks No-long-jump zone!!!
4987	*/
4988	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4989	{
4990	int rc = VINF_SUCCESS;
4991	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
4992	{
4993	RTGCUINTREG uVal = 0;
4994	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4995	pMixedCtx->gs.u64Base = uVal;
4996	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
4997	}
4998	return rc;
4999	}
5000
5001
5002	/**
5003	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5004	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5005	*
5006	* @returns VBox status code.
5007	* @param pVCpu Pointer to the VMCPU.
5008	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5009	* out-of-sync. Make sure to update the required fields
5010	* before using them.
5011	*
5012	* @remarks No-long-jump zone!!!
5013	*/
5014	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5015	{
5016	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5017	return VINF_SUCCESS;
5018
5019	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5020	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5021	{
5022	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5023	pMsr += i;
5024	switch (pMsr->u32IndexMSR)
5025	{
5026	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5027	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5028	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5029	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5030	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5031	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5032	default:
5033	{
5034	AssertFailed();
5035	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5036	}
5037	}
5038	}
5039	#endif
5040
5041	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5042	return VINF_SUCCESS;
5043	}
5044
5045
5046	/**
5047	* Saves the guest control registers from the current VMCS into the guest-CPU
5048	* context.
5049	*
5050	* @returns VBox status code.
5051	* @param pVCpu Pointer to the VMCPU.
5052	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5053	* out-of-sync. Make sure to update the required fields
5054	* before using them.
5055	*
5056	* @remarks No-long-jump zone!!!
5057	*/
5058	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5059	{
5060	/* Guest CR0. Guest FPU. */
5061	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5062
5063	/* Guest CR4. */
5064	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5065	AssertRCReturn(rc, rc);
5066
5067	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5068	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5069	{
5070	PVM pVM = pVCpu->CTX_SUFF(pVM);
5071	if ( pVM->hm.s.fNestedPaging
5072	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5073	{
5074	RTGCUINTREG uVal = 0;
5075	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5076	if (pMixedCtx->cr3 != uVal)
5077	{
5078	CPUMSetGuestCR3(pVCpu, uVal);
5079	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5080	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5081	}
5082
5083	/* We require EFER to check PAE mode. */
5084	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5085
5086	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5087	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5088	{
5089	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5090	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5091	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5092	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5093	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5094	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5095	}
5096	AssertRCReturn(rc, rc);
5097	}
5098
5099	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5100	}
5101	return rc;
5102	}
5103
5104
5105	/**
5106	* Reads a guest segment register from the current VMCS into the guest-CPU
5107	* context.
5108	*
5109	* @returns VBox status code.
5110	* @param pVCpu Pointer to the VMCPU.
5111	* @param idxSel Index of the selector in the VMCS.
5112	* @param idxLimit Index of the segment limit in the VMCS.
5113	* @param idxBase Index of the segment base in the VMCS.
5114	* @param idxAccess Index of the access rights of the segment in the VMCS.
5115	* @param pSelReg Pointer to the segment selector.
5116	*
5117	* @remarks No-long-jump zone!!!
5118	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5119	* as that takes care of whether to read from the VMCS cache or not.
5120	*/
5121	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5122	PCPUMSELREG pSelReg)
5123	{
5124	uint32_t u32Val = 0;
5125	int rc = VMXReadVmcs32(idxSel, &u32Val);
5126	pSelReg->Sel = (uint16_t)u32Val;
5127	pSelReg->ValidSel = (uint16_t)u32Val;
5128	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5129
5130	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5131	pSelReg->u32Limit = u32Val;
5132
5133	RTGCUINTREG uGCVal = 0;
5134	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5135	pSelReg->u64Base = uGCVal;
5136
5137	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5138	pSelReg->Attr.u = u32Val;
5139	AssertRCReturn(rc, rc);
5140
5141	/*
5142	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5143	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5144	*/
5145	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5146	{
5147	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5148	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5149	}
5150	return rc;
5151	}
5152
5153
5154	/**
5155	* Saves the guest segment registers from the current VMCS into the guest-CPU
5156	* context.
5157	*
5158	* @returns VBox status code.
5159	* @param pVCpu Pointer to the VMCPU.
5160	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5161	* out-of-sync. Make sure to update the required fields
5162	* before using them.
5163	*
5164	* @remarks No-long-jump zone!!!
5165	*/
5166	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5167	{
5168	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5169	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5170	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5171	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5172	#else
5173	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5174	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5175	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5176	#endif
5177
5178	int rc = VINF_SUCCESS;
5179
5180	/* Guest segment registers. */
5181	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5182	{
5183	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5184	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5185	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5186	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5187	rc \|= VMXLOCAL_READ_SEG(ES, es);
5188	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5189	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5190	AssertRCReturn(rc, rc);
5191
5192	/* Restore segment attributes for real-on-v86 mode hack. */
5193	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5194	{
5195	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5196	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5197	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5198	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5199	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5200	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5201	}
5202	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5203	}
5204
5205	/* Guest LDTR. */
5206	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5207	{
5208	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5209	AssertRCReturn(rc, rc);
5210	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5211	}
5212
5213	/* Guest GDTR. */
5214	RTGCUINTREG uGCVal = 0;
5215	uint32_t u32Val = 0;
5216	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5217	{
5218	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5219	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5220	pMixedCtx->gdtr.pGdt = uGCVal;
5221	pMixedCtx->gdtr.cbGdt = u32Val;
5222	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5223	}
5224
5225	/* Guest IDTR. */
5226	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5227	{
5228	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5229	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5230	pMixedCtx->idtr.pIdt = uGCVal;
5231	pMixedCtx->idtr.cbIdt = u32Val;
5232	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5233	}
5234
5235	/* Guest TR. */
5236	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5237	{
5238	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5239
5240	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5241	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5242	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5243	AssertRCReturn(rc, rc);
5244	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5245	}
5246	return rc;
5247	}
5248
5249
5250	/**
5251	* Saves the guest debug registers from the current VMCS into the guest-CPU
5252	* context.
5253	*
5254	* @returns VBox status code.
5255	* @param pVCpu Pointer to the VMCPU.
5256	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5257	* out-of-sync. Make sure to update the required fields
5258	* before using them.
5259	*
5260	* @remarks No-long-jump zone!!!
5261	*/
5262	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5263	{
5264	int rc = VINF_SUCCESS;
5265	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5266	{
5267	RTGCUINTREG uVal;
5268	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5269	pMixedCtx->dr[7] = uVal;
5270
5271	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5272	}
5273	return rc;
5274	}
5275
5276
5277	/**
5278	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5279	*
5280	* @returns VBox status code.
5281	* @param pVCpu Pointer to the VMCPU.
5282	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5283	* out-of-sync. Make sure to update the required fields
5284	* before using them.
5285	*
5286	* @remarks No-long-jump zone!!!
5287	*/
5288	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5289	{
5290	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5292	return VINF_SUCCESS;
5293	}
5294
5295
5296	/**
5297	* Saves the entire guest state from the currently active VMCS into the
5298	* guest-CPU context. This essentially VMREADs all guest-data.
5299	*
5300	* @returns VBox status code.
5301	* @param pVCpu Pointer to the VMCPU.
5302	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5303	* out-of-sync. Make sure to update the required fields
5304	* before using them.
5305	*/
5306	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5307	{
5308	Assert(pVCpu);
5309	Assert(pMixedCtx);
5310
5311	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5312	return VINF_SUCCESS;
5313
5314	VMMRZCallRing3Disable(pVCpu);
5315	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5316	LogFunc(("\n"));
5317
5318	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5320
5321	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5323
5324	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5326
5327	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5329
5330	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5332
5333	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5335
5336	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5338
5339	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5341
5342	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5343	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5344
5345	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5346	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5347
5348	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5349	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5350
5351	VMMRZCallRing3Enable(pVCpu);
5352	return rc;
5353	}
5354
5355
5356	/**
5357	* Check per-VM and per-VCPU force flag actions that require us to go back to
5358	* ring-3 for one reason or another.
5359	*
5360	* @returns VBox status code (information status code included).
5361	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5362	* ring-3.
5363	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5364	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5365	* interrupts)
5366	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5367	* all EMTs to be in ring-3.
5368	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5369	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5370	* to the EM loop.
5371	*
5372	* @param pVM Pointer to the VM.
5373	* @param pVCpu Pointer to the VMCPU.
5374	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5375	* out-of-sync. Make sure to update the required fields
5376	* before using them.
5377	*/
5378	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5379	{
5380	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5381
5382	int rc = VERR_INTERNAL_ERROR_5;
5383	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5384	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5385	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5386	{
5387	/* We need the control registers now, make sure the guest-CPU context is updated. */
5388	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5389	AssertRCReturn(rc, rc);
5390
5391	/* Pending HM CR3 sync. */
5392	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5393	{
5394	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5395	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5396	}
5397	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5398	{
5399	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5400	AssertRC(rc);
5401	}
5402
5403	/* Pending PGM C3 sync. */
5404	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5405	{
5406	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5407	if (rc != VINF_SUCCESS)
5408	{
5409	AssertRC(rc);
5410	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5411	return rc;
5412	}
5413	}
5414
5415	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5416	/* -XXX- what was that about single stepping? */
5417	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5418	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5419	{
5420	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5421	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5422	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5423	return rc;
5424	}
5425
5426	/* Pending VM request packets, such as hardware interrupts. */
5427	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5428	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5429	{
5430	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5431	return VINF_EM_PENDING_REQUEST;
5432	}
5433
5434	/* Pending PGM pool flushes. */
5435	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5436	{
5437	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5438	return VINF_PGM_POOL_FLUSH_PENDING;
5439	}
5440
5441	/* Pending DMA requests. */
5442	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5443	{
5444	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5445	return VINF_EM_RAW_TO_R3;
5446	}
5447	}
5448
5449	/* Paranoia. */
5450	Assert(rc != VERR_EM_INTERPRETER);
5451	return VINF_SUCCESS;
5452	}
5453
5454
5455	/**
5456	* Converts any TRPM trap into a pending VMX event. This is typically used when
5457	* entering from ring-3 (not longjmp returns).
5458	*
5459	* @param pVCpu Pointer to the VMCPU.
5460	* @param pCtx Pointer to the guest-CPU context.
5461	*/
5462	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5463	{
5464	if (!TRPMHasTrap(pVCpu))
5465	{
5466	Assert(!pVCpu->hm.s.Event.fPending);
5467	return;
5468	}
5469
5470	uint8_t uVector;
5471	TRPMEVENT enmTrpmEvent;
5472	RTGCUINT uErrCode;
5473	RTGCUINTPTR GCPtrFaultAddress;
5474	uint8_t cbInstr;
5475
5476	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5477	AssertRC(rc);
5478
5479	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5480	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5481	if (enmTrpmEvent == TRPM_TRAP)
5482	{
5483	switch (uVector)
5484	{
5485	case X86_XCPT_BP:
5486	case X86_XCPT_OF:
5487	{
5488	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5489	break;
5490	}
5491
5492	case X86_XCPT_PF:
5493	case X86_XCPT_DF:
5494	case X86_XCPT_TS:
5495	case X86_XCPT_NP:
5496	case X86_XCPT_SS:
5497	case X86_XCPT_GP:
5498	case X86_XCPT_AC:
5499	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5500	/* no break! */
5501	default:
5502	{
5503	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5504	break;
5505	}
5506	}
5507	}
5508	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5509	{
5510	if (uVector != X86_XCPT_NMI)
5511	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5512	else
5513	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5514	}
5515	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5516	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5517	else
5518	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5519
5520	rc = TRPMResetTrap(pVCpu);
5521	AssertRC(rc);
5522	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5523	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5524	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5525	}
5526
5527
5528	/**
5529	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5530	* VT-x to execute any instruction.
5531	*
5532	* @param pvCpu Pointer to the VMCPU.
5533	*/
5534	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5535	{
5536	if (pVCpu->hm.s.Event.fPending)
5537	{
5538	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5539	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5540	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5541	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5542
5543	/* If a trap was already pending, we did something wrong! */
5544	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5545
5546	TRPMEVENT enmTrapType;
5547	switch (uVectorType)
5548	{
5549	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5550	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5551	enmTrapType = TRPM_HARDWARE_INT;
5552	break;
5553	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5554	enmTrapType = TRPM_SOFTWARE_INT;
5555	break;
5556	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5557	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5558	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5559	enmTrapType = TRPM_TRAP;
5560	break;
5561	default:
5562	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5563	enmTrapType = TRPM_32BIT_HACK;
5564	break;
5565	}
5566
5567	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5568	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5569	AssertRC(rc);
5570
5571	if (fErrorCodeValid)
5572	TRPMSetErrorCode(pVCpu, uErrorCode);
5573	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5574	&& uVector == X86_XCPT_PF)
5575	{
5576	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5577	}
5578	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5579	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5580	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5581	{
5582	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5583	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5584	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5585	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5586	}
5587	pVCpu->hm.s.Event.fPending = false;
5588	}
5589	}
5590
5591
5592	/**
5593	* Does the necessary state syncing before doing a longjmp to ring-3.
5594	*
5595	* @param pVM Pointer to the VM.
5596	* @param pVCpu Pointer to the VMCPU.
5597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5598	* out-of-sync. Make sure to update the required fields
5599	* before using them.
5600	* @param rcExit The reason for exiting to ring-3. Can be
5601	* VINF_VMM_UNKNOWN_RING3_CALL.
5602	*
5603	* @remarks No-long-jmp zone!!!
5604	*/
5605	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5606	{
5607	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5608	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5609
5610	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5611	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5612	AssertRC(rc);
5613
5614	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5615	if (CPUMIsGuestFPUStateActive(pVCpu))
5616	{
5617	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5618	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5619	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5620	}
5621
5622	/* Restore debug registers if necessary and resync on next R0 reentry. */
5623	if (CPUMIsGuestDebugStateActive(pVCpu))
5624	{
5625	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5626	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5627	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5628	}
5629	else if (CPUMIsHyperDebugStateActive(pVCpu))
5630	{
5631	CPUMR0LoadHostDebugState(pVM, pVCpu);
5632	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5633	}
5634
5635	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5636	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5637	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5638	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5639	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5640	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5641	}
5642
5643
5644	/**
5645	* An action requires us to go back to ring-3. This function does the necessary
5646	* steps before we can safely return to ring-3. This is not the same as longjmps
5647	* to ring-3, this is voluntary.
5648	*
5649	* @param pVM Pointer to the VM.
5650	* @param pVCpu Pointer to the VMCPU.
5651	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5652	* out-of-sync. Make sure to update the required fields
5653	* before using them.
5654	* @param rcExit The reason for exiting to ring-3. Can be
5655	* VINF_VMM_UNKNOWN_RING3_CALL.
5656	*/
5657	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5658	{
5659	Assert(pVM);
5660	Assert(pVCpu);
5661	Assert(pMixedCtx);
5662	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5663
5664	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5665	{
5666	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5667	return;
5668	}
5669	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5670	{
5671	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5672	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5673	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5674	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5675	return;
5676	}
5677
5678	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5679	VMMRZCallRing3Disable(pVCpu);
5680	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5681
5682	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5683	hmR0VmxUpdateTRPM(pVCpu);
5684
5685	/* Sync. the guest state. */
5686	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5687	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5688
5689	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5690	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5691	\| CPUM_CHANGED_LDTR
5692	\| CPUM_CHANGED_GDTR
5693	\| CPUM_CHANGED_IDTR
5694	\| CPUM_CHANGED_TR
5695	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5696
5697	/* On our way back from ring-3 the following needs to be done. */
5698	/** @todo This can change with preemption hooks. */
5699	if (rcExit == VINF_EM_RAW_INTERRUPT)
5700	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5701	else
5702	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5703
5704	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5705	VMMRZCallRing3Enable(pVCpu);
5706	}
5707
5708
5709	/**
5710	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5711	* longjump to ring-3 and possibly get preempted.
5712	*
5713	* @param pVCpu Pointer to the VMCPU.
5714	* @param enmOperation The operation causing the ring-3 longjump.
5715	* @param pvUser The user argument (pointer to the possibly
5716	* out-of-date guest-CPU context).
5717	*
5718	* @remarks Must never be called with @a enmOperation ==
5719	* VMMCALLRING3_VM_R0_ASSERTION.
5720	*/
5721	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5722	{
5723	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5724	Assert(pVCpu);
5725	Assert(pvUser);
5726	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5727	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5728
5729	VMMRZCallRing3Disable(pVCpu);
5730	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5731	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5732	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5733	VMMRZCallRing3Enable(pVCpu);
5734	}
5735
5736
5737	/**
5738	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5739	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5740	*
5741	* @returns VBox status code.
5742	* @param pVCpu Pointer to the VMCPU.
5743	*/
5744	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5745	{
5746	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5747	{
5748	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5749	{
5750	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5751	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5752	AssertRC(rc);
5753	}
5754	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5755	}
5756
5757
5758	/**
5759	* Injects any pending events into the guest if the guest is in a state to
5760	* receive them.
5761	*
5762	* @returns VBox status code (informational status codes included).
5763	* @param pVCpu Pointer to the VMCPU.
5764	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5765	* out-of-sync. Make sure to update the required fields
5766	* before using them.
5767	*/
5768	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5769	{
5770	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5771	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5772	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5773	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5774
5775	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5776	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5777	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5778	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5779
5780	int rc = VINF_SUCCESS;
5781	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5782	{
5783	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5784	bool fInject = true;
5785	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5786	{
5787	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5788	AssertRCReturn(rc, rc);
5789	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5790	if ( fBlockInt
5791	\|\| fBlockSti
5792	\|\| fBlockMovSS)
5793	{
5794	fInject = false;
5795	}
5796	}
5797	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5798	&& ( fBlockMovSS
5799	\|\| fBlockSti))
5800	{
5801	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5802	fInject = false;
5803	}
5804
5805	if (fInject)
5806	{
5807	Log(("Injecting pending event\n"));
5808	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5809	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5810	AssertRCReturn(rc, rc);
5811	pVCpu->hm.s.Event.fPending = false;
5812	}
5813	else
5814	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5815	} /** @todo SMI. SMIs take priority over NMIs. */
5816	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5817	{
5818	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5819	if ( !fBlockMovSS
5820	&& !fBlockSti)
5821	{
5822	Log(("Injecting NMI\n"));
5823	RTGCUINTPTR uIntrInfo;
5824	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5825	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5826	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5827	0 /* GCPtrFaultAddress */, &uIntrState);
5828	AssertRCReturn(rc, rc);
5829	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5830	}
5831	else
5832	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5833	}
5834	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5835	{
5836	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5837	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5838	AssertRCReturn(rc, rc);
5839	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5840	if ( !fBlockInt
5841	&& !fBlockSti
5842	&& !fBlockMovSS)
5843	{
5844	uint8_t u8Interrupt;
5845	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5846	if (RT_SUCCESS(rc))
5847	{
5848	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5849	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5850	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5851	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5852	0 /* GCPtrFaultAddress */, &uIntrState);
5853	}
5854	else
5855	{
5856	/** @todo Does this actually happen? If not turn it into an assertion. */
5857	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5858	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5859	rc = VINF_SUCCESS;
5860	}
5861	}
5862	else
5863	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5864	}
5865
5866	/*
5867	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5868	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5869	*/
5870	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5871	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5872	int rc2 = VINF_SUCCESS;
5873	if ( fBlockSti
5874	\|\| fBlockMovSS)
5875	{
5876	if (!DBGFIsStepping(pVCpu))
5877	{
5878	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5879	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5880	{
5881	/*
5882	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5883	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5884	*/
5885	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5886	}
5887	}
5888	else
5889	{
5890	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5891	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5892	uIntrState = 0;
5893	}
5894	}
5895
5896	/*
5897	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5898	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5899	*/
5900	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5901	AssertRC(rc2);
5902
5903	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5904	return rc;
5905	}
5906
5907
5908	/**
5909	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5910	*
5911	* @param pVCpu Pointer to the VMCPU.
5912	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5913	* out-of-sync. Make sure to update the required fields
5914	* before using them.
5915	*/
5916	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5917	{
5918	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5919	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5920	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5921	}
5922
5923
5924	/**
5925	* Injects a double-fault (#DF) exception into the VM.
5926	*
5927	* @returns VBox status code (informational status code included).
5928	* @param pVCpu Pointer to the VMCPU.
5929	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5930	* out-of-sync. Make sure to update the required fields
5931	* before using them.
5932	*/
5933	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5934	{
5935	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5936	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5937	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5938	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5939	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5940	puIntrState);
5941	}
5942
5943
5944	/**
5945	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5946	*
5947	* @param pVCpu Pointer to the VMCPU.
5948	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5949	* out-of-sync. Make sure to update the required fields
5950	* before using them.
5951	*/
5952	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5953	{
5954	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5955	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5956	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5957	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5958	}
5959
5960
5961	/**
5962	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
5963	*
5964	* @param pVCpu Pointer to the VMCPU.
5965	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5966	* out-of-sync. Make sure to update the required fields
5967	* before using them.
5968	* @param cbInstr The value of RIP that is to be pushed on the guest
5969	* stack.
5970	*/
5971	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5972	{
5973	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5974	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5975	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5976	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
5977	}
5978
5979
5980	/**
5981	* Injects a general-protection (#GP) fault into the VM.
5982	*
5983	* @returns VBox status code (informational status code included).
5984	* @param pVCpu Pointer to the VMCPU.
5985	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5986	* out-of-sync. Make sure to update the required fields
5987	* before using them.
5988	* @param u32ErrorCode The error code associated with the #GP.
5989	*/
5990	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
5991	uint32_t *puIntrState)
5992	{
5993	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5994	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5995	if (fErrorCodeValid)
5996	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5997	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5998	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
5999	puIntrState);
6000	}
6001
6002
6003	/**
6004	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6005	*
6006	* @param pVCpu Pointer to the VMCPU.
6007	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6008	* out-of-sync. Make sure to update the required fields
6009	* before using them.
6010	* @param uVector The software interrupt vector number.
6011	* @param cbInstr The value of RIP that is to be pushed on the guest
6012	* stack.
6013	*/
6014	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6015	{
6016	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6017	if ( uVector == X86_XCPT_BP
6018	\|\| uVector == X86_XCPT_OF)
6019	{
6020	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6021	}
6022	else
6023	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6024	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6025	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6026	}
6027
6028
6029	/**
6030	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6031	* stack.
6032	*
6033	* @returns VBox status code (information status code included).
6034	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6035	* @param pVM Pointer to the VM.
6036	* @param pMixedCtx Pointer to the guest-CPU context.
6037	* @param uValue The value to push to the guest stack.
6038	*/
6039	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6040	{
6041	/*
6042	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6043	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6044	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6045	*/
6046	if (pMixedCtx->sp == 1)
6047	return VINF_EM_RESET;
6048	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6049	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6050	AssertRCReturn(rc, rc);
6051	return rc;
6052	}
6053
6054
6055	/**
6056	* Injects an event into the guest upon VM-entry by updating the relevant fields
6057	* in the VM-entry area in the VMCS.
6058	*
6059	* @returns VBox status code (informational error codes included).
6060	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6061	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6062	*
6063	* @param pVCpu Pointer to the VMCPU.
6064	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6065	* be out-of-sync. Make sure to update the required
6066	* fields before using them.
6067	* @param u64IntrInfo The VM-entry interruption-information field.
6068	* @param cbInstr The VM-entry instruction length in bytes (for
6069	* software interrupts, exceptions and privileged
6070	* software exceptions).
6071	* @param u32ErrCode The VM-entry exception error code.
6072	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6073	* @param puIntrState Pointer to the current guest interruptibility-state.
6074	* This interruptibility-state will be updated if
6075	* necessary. This cannot not be NULL.
6076	*
6077	* @remarks No-long-jump zone!!!
6078	* @remarks Requires CR0!
6079	*/
6080	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6081	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6082	{
6083	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6084	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6085	Assert(puIntrState);
6086	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6087
6088	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6089	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6090
6091	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6092	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6093	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6094
6095	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6096
6097	/* We require CR0 to check if the guest is in real-mode. */
6098	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6099	AssertRCReturn(rc, rc);
6100
6101	/*
6102	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6103	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6104	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6105	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6106	*/
6107	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6108	{
6109	PVM pVM = pVCpu->CTX_SUFF(pVM);
6110	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6111	{
6112	Assert(PDMVmmDevHeapIsEnabled(pVM));
6113	Assert(pVM->hm.s.vmx.pRealModeTSS);
6114
6115	/* Save the required guest state bits from the VMCS. */
6116	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6117	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6118	AssertRCReturn(rc, rc);
6119
6120	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6121	const size_t cbIdtEntry = 4;
6122	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6123	{
6124	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6125	if (uVector == X86_XCPT_DF)
6126	return VINF_EM_RESET;
6127	else if (uVector == X86_XCPT_GP)
6128	{
6129	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6130	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6131	}
6132
6133	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6134	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6135	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6136	}
6137
6138	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6139	uint16_t uGuestIp = pMixedCtx->ip;
6140	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6141	{
6142	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6143	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6144	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6145	}
6146	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6147	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6148
6149	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6150	uint16_t offIdtEntry = 0;
6151	RTSEL selIdtEntry = 0;
6152	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6153	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6154	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6155	AssertRCReturn(rc, rc);
6156
6157	/* Construct the stack frame for the interrupt/exception handler. */
6158	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6159	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6160	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6161	AssertRCReturn(rc, rc);
6162
6163	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6164	if (rc == VINF_SUCCESS)
6165	{
6166	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6167	pMixedCtx->rip = offIdtEntry;
6168	pMixedCtx->cs.Sel = selIdtEntry;
6169	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6170	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6171	&& uVector == X86_XCPT_PF)
6172	{
6173	pMixedCtx->cr2 = GCPtrFaultAddress;
6174	}
6175	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6176	\| HM_CHANGED_GUEST_RIP
6177	\| HM_CHANGED_GUEST_RFLAGS
6178	\| HM_CHANGED_GUEST_RSP;
6179
6180	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6181	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6182	{
6183	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6184	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6185	Log(("Clearing inhibition due to STI.\n"));
6186	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6187	}
6188	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6189	}
6190	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6191	return rc;
6192	}
6193	else
6194	{
6195	/*
6196	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6197	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6198	*/
6199	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6200	}
6201	}
6202
6203	/* Validate. */
6204	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6205	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6206	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6207
6208	/* Inject. */
6209	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6210	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6211	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6212	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6213
6214	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6215	&& uVector == X86_XCPT_PF)
6216	{
6217	pMixedCtx->cr2 = GCPtrFaultAddress;
6218	}
6219	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6220
6221	AssertRCReturn(rc, rc);
6222	return rc;
6223	}
6224
6225
6226	/**
6227	* Enters the VT-x session.
6228	*
6229	* @returns VBox status code.
6230	* @param pVM Pointer to the VM.
6231	* @param pVCpu Pointer to the VMCPU.
6232	* @param pCpu Pointer to the CPU info struct.
6233	*/
6234	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6235	{
6236	AssertPtr(pVM);
6237	AssertPtr(pVCpu);
6238	Assert(pVM->hm.s.vmx.fSupported);
6239	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6240	NOREF(pCpu);
6241
6242	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6243
6244	/* Make sure we're in VMX root mode. */
6245	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6246	if (!(u32HostCR4 & X86_CR4_VMXE))
6247	{
6248	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6249	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6250	}
6251
6252	/* Load the active VMCS as the current one. */
6253	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6254	if (RT_FAILURE(rc))
6255	return rc;
6256
6257	/** @todo this will change with preemption hooks where can can VMRESUME as long
6258	* as we're no preempted. */
6259	pVCpu->hm.s.fResumeVM = false;
6260	return VINF_SUCCESS;
6261	}
6262
6263
6264	/**
6265	* Leaves the VT-x session.
6266	*
6267	* @returns VBox status code.
6268	* @param pVM Pointer to the VM.
6269	* @param pVCpu Pointer to the VMCPU.
6270	* @param pCtx Pointer to the guest-CPU context.
6271	*/
6272	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6273	{
6274	AssertPtr(pVCpu);
6275	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6276	NOREF(pVM);
6277	NOREF(pCtx);
6278
6279	/** @todo this will change with preemption hooks where we only VMCLEAR when
6280	* we are actually going to be preempted, not all the time like we
6281	* currently do. */
6282	/*
6283	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6284	* and mark the VMCS launch-state as "clear".
6285	*/
6286	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6287	return rc;
6288	}
6289
6290
6291	/**
6292	* Saves the host state in the VMCS host-state.
6293	* Sets up the VM-exit MSR-load area.
6294	*
6295	* The CPU state will be loaded from these fields on every successful VM-exit.
6296	*
6297	* @returns VBox status code.
6298	* @param pVM Pointer to the VM.
6299	* @param pVCpu Pointer to the VMCPU.
6300	*
6301	* @remarks No-long-jump zone!!!
6302	*/
6303	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6304	{
6305	AssertPtr(pVM);
6306	AssertPtr(pVCpu);
6307	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6308
6309	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6310
6311	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6312	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6313	return VINF_SUCCESS;
6314
6315	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6320
6321	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6323
6324	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6325	return rc;
6326	}
6327
6328
6329	/**
6330	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6331	* loaded from these fields on every successful VM-entry.
6332	*
6333	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6334	* Sets up the VM-entry controls.
6335	* Sets up the appropriate VMX non-root function to execute guest code based on
6336	* the guest CPU mode.
6337	*
6338	* @returns VBox status code.
6339	* @param pVM Pointer to the VM.
6340	* @param pVCpu Pointer to the VMCPU.
6341	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6342	* out-of-sync. Make sure to update the required fields
6343	* before using them.
6344	*
6345	* @remarks No-long-jump zone!!!
6346	*/
6347	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6348	{
6349	AssertPtr(pVM);
6350	AssertPtr(pVCpu);
6351	AssertPtr(pMixedCtx);
6352	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6353
6354	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6355
6356	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6357
6358	/* Determine real-on-v86 mode. */
6359	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6360	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6361	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6362	{
6363	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6364	}
6365
6366	/** @todo if the order of loading is important, inform it via comments here */
6367	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6368	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6369
6370	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6371	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6372
6373	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6374	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6375
6376	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6377	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6378
6379	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6380	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6381
6382	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6384
6385	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6387
6388	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6389	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6390
6391	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6392	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6393
6394	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6395	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6396
6397	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6398	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6399	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6400
6401	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6402	return rc;
6403	}
6404
6405
6406	/**
6407	* Does the preparations before executing guest code in VT-x.
6408	*
6409	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6410	* recompiler. We must be cautious what we do here regarding committing
6411	* guest-state information into the the VMCS assuming we assuredly execute the
6412	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6413	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6414	* that the recompiler can (and should) use them when it resumes guest
6415	* execution. Otherwise such operations must be done when we can no longer
6416	* exit to ring-3.
6417	*
6418	* @returns VBox status code (informational status codes included).
6419	* @retval VINF_SUCCESS if we can proceed with running the guest.
6420	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6421	* into the guest.
6422	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6423	*
6424	* @param pVM Pointer to the VM.
6425	* @param pVCpu Pointer to the VMCPU.
6426	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6427	* out-of-sync. Make sure to update the required fields
6428	* before using them.
6429	* @param pVmxTransient Pointer to the VMX transient structure.
6430	*
6431	* @remarks Called with preemption disabled.
6432	*/
6433	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6434	{
6435	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6436
6437	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6438	PGMRZDynMapFlushAutoSet(pVCpu);
6439	#endif
6440
6441	/* Check force flag actions that might require us to go back to ring-3. */
6442	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6443	if (rc != VINF_SUCCESS)
6444	return rc;
6445
6446	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6447	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6448	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6449	{
6450	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6451	RTGCPHYS GCPhysApicBase;
6452	GCPhysApicBase = pMixedCtx->msrApicBase;
6453	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6454
6455	/* Unalias any existing mapping. */
6456	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6457	AssertRCReturn(rc, rc);
6458
6459	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6460	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6461	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6462	AssertRCReturn(rc, rc);
6463
6464	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6465	}
6466
6467	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6468	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6469	pVmxTransient->uEFlags = ASMIntDisableFlags();
6470	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6471	{
6472	ASMSetFlags(pVmxTransient->uEFlags);
6473	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6474	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6475	return VINF_EM_RAW_INTERRUPT;
6476	}
6477	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6478	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6479	#endif
6480
6481	/*
6482	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6483	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6484	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6485	*/
6486	/** @todo r=bird: You reverse the effect of calling PDMGetInterrupt by
6487	* handing it over to TPRM like we do in REMR3StateBack using
6488	* TRPMAssertTrap and the other setters. */
6489	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6490	AssertRCReturn(rc, rc);
6491	return rc;
6492	}
6493
6494
6495	/**
6496	* Prepares to run guest code in VT-x and we've committed to doing so. This
6497	* means there is no backing out to ring-3 or anywhere else at this
6498	* point.
6499	*
6500	* @param pVM Pointer to the VM.
6501	* @param pVCpu Pointer to the VMCPU.
6502	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6503	* out-of-sync. Make sure to update the required fields
6504	* before using them.
6505	* @param pVmxTransient Pointer to the VMX transient structure.
6506	*
6507	* @remarks Called with preemption disabled.
6508	* @remarks No-long-jump zone!!!
6509	*/
6510	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6511	{
6512	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6513	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6514
6515	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6516	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6517	pVmxTransient->uEFlags = ASMIntDisableFlags();
6518	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6519	#endif
6520
6521	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6522	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6523	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6524	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6525	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6526	#endif
6527	int rc = VINF_SUCCESS;
6528	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6529	{
6530	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6531	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6532	}
6533	else if (pVCpu->hm.s.fContextUseFlags)
6534	{
6535	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6536	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6537	}
6538	AssertRC(rc);
6539	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6540
6541	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6542	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6543	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6544
6545	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6546	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6547	{
6548	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6549	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6550	}
6551
6552	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6553	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6554	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6555
6556	/*
6557	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6558	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6559	*/
6560	if (pVM->hm.s.fTPRPatchingActive)
6561	{
6562	Assert(!CPUMIsGuestInLongMode(pVCpu));
6563
6564	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6565	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6566	AssertRC(rc);
6567
6568	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6569	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6570	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6571	see hmR0VmxLoadGuestApicState(). */
6572	}
6573
6574	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6575	/*
6576	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6577	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6578	*/
6579	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6580	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6581	{
6582	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6583	uint64_t u64HostTscAux = 0;
6584	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6585	AssertRC(rc2);
6586	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6587	}
6588	#endif
6589
6590	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6591	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6592	to start executing. */
6593	}
6594
6595
6596	/**
6597	* Performs some essential restoration of state after running guest code in
6598	* VT-x.
6599	*
6600	* @param pVM Pointer to the VM.
6601	* @param pVCpu Pointer to the VMCPU.
6602	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6603	* out-of-sync. Make sure to update the required fields
6604	* before using them.
6605	* @param pVmxTransient Pointer to the VMX transient structure.
6606	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6607	*
6608	* @remarks Called with interrupts disabled.
6609	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6610	* unconditionally when it is safe to do so.
6611	*/
6612	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6613	{
6614	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6615	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6616
6617	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6618	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6619	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6620	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6621	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6622
6623	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6624	{
6625	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6626	/* Restore host's TSC_AUX. */
6627	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6628	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6629	#endif
6630	/** @todo Find a way to fix hardcoding a guestimate. */
6631	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6632	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6633	}
6634
6635	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6636	Assert(!(ASMGetFlags() & X86_EFL_IF));
6637	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6638
6639	/* Restore the effects of TPR patching if any. */
6640	if (pVM->hm.s.fTPRPatchingActive)
6641	{
6642	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6643	AssertRC(rc);
6644	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6645	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6646	}
6647
6648	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6649	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6650
6651	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6652	uint32_t uExitReason;
6653	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6654	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6655	AssertRC(rc);
6656	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6657	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6658
6659	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6660	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6661
6662	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6663	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6664	{
6665	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6666	return;
6667	}
6668
6669	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6670	{
6671	/* Update the guest interruptibility-state from the VMCS. */
6672	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6673	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6674	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6675	AssertRC(rc);
6676	#endif
6677	/*
6678	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6679	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6680	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6681	*/
6682	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6683	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6684	{
6685	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6686	AssertRC(rc);
6687	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6688	}
6689	}
6690	}
6691
6692
6693	/**
6694	* Runs the guest code using VT-x.
6695	*
6696	* @returns VBox status code.
6697	* @param pVM Pointer to the VM.
6698	* @param pVCpu Pointer to the VMCPU.
6699	* @param pCtx Pointer to the guest-CPU context.
6700	*
6701	* @remarks Called with preemption disabled.
6702	*/
6703	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6704	{
6705	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6706	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6707
6708	VMXTRANSIENT VmxTransient;
6709	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6710	int rc = VERR_INTERNAL_ERROR_5;
6711	uint32_t cLoops = 0;
6712	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6713
6714	for (;; cLoops++)
6715	{
6716	Assert(!HMR0SuspendPending());
6717	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6718	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6719	(unsigned)RTMpCpuId(), cLoops));
6720
6721	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6722	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6723	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6724	if (rc != VINF_SUCCESS)
6725	break;
6726
6727	/*
6728	* No longjmps to ring-3 from this point on!!!
6729	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6730	* This also disables flushing of the R0-logger instance (if any).
6731	*/
6732	VMMRZCallRing3Disable(pVCpu);
6733	VMMRZCallRing3RemoveNotification(pVCpu);
6734	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6735
6736	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6737	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6738
6739	/*
6740	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6741	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6742	*/
6743	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6744	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6745	{
6746	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6747	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6748	return rc;
6749	}
6750
6751	/* Handle the VM-exit. */
6752	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6753	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6754	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6755	#ifdef HM_PROFILE_EXIT_DISPATCH
6756	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6757	#endif
6758	#ifdef HMVMX_USE_FUNCTION_TABLE
6759	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6760	#else
6761	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6762	#endif
6763	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6764	if (rc != VINF_SUCCESS)
6765	break;
6766	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6767	{
6768	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6769	rc = VINF_EM_RAW_INTERRUPT;
6770	break;
6771	}
6772	}
6773
6774	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6775	if (rc == VERR_EM_INTERPRETER)
6776	rc = VINF_EM_RAW_EMULATE_INSTR;
6777	else if (rc == VINF_EM_RESET)
6778	rc = VINF_EM_TRIPLE_FAULT;
6779	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6780	return rc;
6781	}
6782
6783
6784	#ifndef HMVMX_USE_FUNCTION_TABLE
6785	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6786	{
6787	int rc;
6788	switch (rcReason)
6789	{
6790	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6791	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6792	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6793	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6794	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6795	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6796	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6797	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6798	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6799	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6800	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6801	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6802	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6803	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6804	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6805	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6806	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6807	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6808	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6809	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6811	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6812	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6813	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6814	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6815	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6820	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6823
6824	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6833
6834	case VMX_EXIT_VMCALL:
6835	case VMX_EXIT_VMCLEAR:
6836	case VMX_EXIT_VMLAUNCH:
6837	case VMX_EXIT_VMPTRLD:
6838	case VMX_EXIT_VMPTRST:
6839	case VMX_EXIT_VMREAD:
6840	case VMX_EXIT_VMRESUME:
6841	case VMX_EXIT_VMWRITE:
6842	case VMX_EXIT_VMXOFF:
6843	case VMX_EXIT_VMXON:
6844	case VMX_EXIT_INVEPT:
6845	case VMX_EXIT_INVVPID:
6846	case VMX_EXIT_VMFUNC:
6847	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6848	break;
6849	default:
6850	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6851	break;
6852	}
6853	return rc;
6854	}
6855	#endif
6856
6857
6858	/** Profiling macro. */
6859	#ifdef HM_PROFILE_EXIT_DISPATCH
6860	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6861	#else
6862	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6863	#endif
6864
6865
6866	#ifdef DEBUG
6867	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6868	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6869	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6870	# define VMX_ASSERT_PREEMPT_CPUID() \
6871	do \
6872	{ \
6873	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6874	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6875	} while (0)
6876
6877	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6878	do { \
6879	AssertPtr(pVCpu); \
6880	AssertPtr(pMixedCtx); \
6881	AssertPtr(pVmxTransient); \
6882	Assert(pVmxTransient->fVMEntryFailed == false); \
6883	Assert(ASMIntAreEnabled()); \
6884	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6885	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6886	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6887	(unsigned)pVCpu->idCpu)); \
6888	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6889	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6890	VMX_ASSERT_PREEMPT_CPUID(); \
6891	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6892	} while (0)
6893	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6894	do { \
6895	LogFunc(("\n")); \
6896	} while(0)
6897	#else /* Release builds */
6898	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6899	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6900	#endif
6901
6902
6903	/**
6904	* Advances the guest RIP after reading it from the VMCS.
6905	*
6906	* @returns VBox status code.
6907	* @param pVCpu Pointer to the VMCPU.
6908	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6909	* out-of-sync. Make sure to update the required fields
6910	* before using them.
6911	* @param pVmxTransient Pointer to the VMX transient structure.
6912	*
6913	* @remarks No-long-jump zone!!!
6914	*/
6915	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6916	{
6917	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6918	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6919	AssertRCReturn(rc, rc);
6920
6921	pMixedCtx->rip += pVmxTransient->cbInstr;
6922	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6923	return rc;
6924	}
6925
6926
6927	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6928	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6929	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6930	/**
6931	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6932	*/
6933	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6934	{
6935	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6936	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6937	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6938	Assert(ASMIntAreEnabled());
6939	return VINF_SUCCESS;
6940	#else
6941	return VINF_EM_RAW_INTERRUPT;
6942	#endif
6943	}
6944
6945
6946	/**
6947	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6948	*/
6949	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6950	{
6951	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6952	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6953
6954	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6955	AssertRCReturn(rc, rc);
6956
6957	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6958	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6959	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6960
6961	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6962	{
6963	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6964	return VINF_EM_RAW_INTERRUPT;
6965	}
6966
6967	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6968	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6969	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6970	{
6971	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6972	return VINF_SUCCESS;
6973	}
6974	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6975	{
6976	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6977	return rc;
6978	}
6979
6980	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6981	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6982	switch (uIntrType)
6983	{
6984	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6985	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6986	/* no break */
6987	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
6988	{
6989	switch (uVector)
6990	{
6991	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6992	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6993	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6994	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6995	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6996	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
6997	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6998	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6999	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7000	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7001	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7002	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7003	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7004	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7005	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7006	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7007	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7008	#endif
7009	default:
7010	{
7011	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7012	AssertRCReturn(rc, rc);
7013
7014	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7015	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7016	{
7017	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7018	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7019	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7020	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7021	AssertRCReturn(rc, rc);
7022	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7023	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7024	0 /* GCPtrFaultAddress */);
7025	AssertRCReturn(rc, rc);
7026	}
7027	else
7028	{
7029	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7030	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7031	}
7032	break;
7033	}
7034	}
7035	break;
7036	}
7037
7038	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7039	default:
7040	{
7041	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7042	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7043	break;
7044	}
7045	}
7046	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7047	return rc;
7048	}
7049
7050
7051	/**
7052	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7053	*/
7054	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7055	{
7056	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7057
7058	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7059	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7060	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7061	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7062	AssertRCReturn(rc, rc);
7063
7064	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7066	return VINF_SUCCESS;
7067	}
7068
7069
7070	/**
7071	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7072	*/
7073	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7074	{
7075	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7076	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7077	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7078	}
7079
7080
7081	/**
7082	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7083	*/
7084	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7085	{
7086	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7087	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7088	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7089	}
7090
7091
7092	/**
7093	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7094	*/
7095	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7096	{
7097	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7099	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7100	}
7101
7102
7103	/**
7104	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7105	*/
7106	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7107	{
7108	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7109	PVM pVM = pVCpu->CTX_SUFF(pVM);
7110	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7111	if (RT_LIKELY(rc == VINF_SUCCESS))
7112	{
7113	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7114	Assert(pVmxTransient->cbInstr == 2);
7115	}
7116	else
7117	{
7118	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7119	rc = VERR_EM_INTERPRETER;
7120	}
7121	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7122	return rc;
7123	}
7124
7125
7126	/**
7127	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7128	*/
7129	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7130	{
7131	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7132	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7133	AssertRCReturn(rc, rc);
7134
7135	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7136	return VINF_EM_RAW_EMULATE_INSTR;
7137
7138	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7139	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7140	}
7141
7142
7143	/**
7144	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7145	*/
7146	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7147	{
7148	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7149	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7150	AssertRCReturn(rc, rc);
7151
7152	PVM pVM = pVCpu->CTX_SUFF(pVM);
7153	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7154	if (RT_LIKELY(rc == VINF_SUCCESS))
7155	{
7156	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7157	Assert(pVmxTransient->cbInstr == 2);
7158	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7159	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7160	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7161	}
7162	else
7163	{
7164	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7165	rc = VERR_EM_INTERPRETER;
7166	}
7167	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7168	return rc;
7169	}
7170
7171
7172	/**
7173	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7174	*/
7175	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7176	{
7177	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7178	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7179	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7180	AssertRCReturn(rc, rc);
7181
7182	PVM pVM = pVCpu->CTX_SUFF(pVM);
7183	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7184	if (RT_LIKELY(rc == VINF_SUCCESS))
7185	{
7186	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7187	Assert(pVmxTransient->cbInstr == 3);
7188	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7189	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7190	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7191	}
7192	else
7193	{
7194	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7195	rc = VERR_EM_INTERPRETER;
7196	}
7197	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7198	return rc;
7199	}
7200
7201
7202	/**
7203	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7204	*/
7205	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7206	{
7207	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7208	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7209	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7210	AssertRCReturn(rc, rc);
7211
7212	PVM pVM = pVCpu->CTX_SUFF(pVM);
7213	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7214	if (RT_LIKELY(rc == VINF_SUCCESS))
7215	{
7216	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7217	Assert(pVmxTransient->cbInstr == 2);
7218	}
7219	else
7220	{
7221	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7222	rc = VERR_EM_INTERPRETER;
7223	}
7224	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7225	return rc;
7226	}
7227
7228
7229	/**
7230	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7231	*/
7232	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7233	{
7234	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7235	PVM pVM = pVCpu->CTX_SUFF(pVM);
7236	Assert(!pVM->hm.s.fNestedPaging);
7237
7238	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7239	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7240	AssertRCReturn(rc, rc);
7241
7242	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7243	rc = VBOXSTRICTRC_VAL(rc2);
7244	if (RT_LIKELY(rc == VINF_SUCCESS))
7245	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7246	else
7247	{
7248	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7249	pVmxTransient->uExitQualification, rc));
7250	}
7251	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7252	return rc;
7253	}
7254
7255
7256	/**
7257	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7258	*/
7259	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7260	{
7261	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7262	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7263	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7264	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7265	AssertRCReturn(rc, rc);
7266
7267	PVM pVM = pVCpu->CTX_SUFF(pVM);
7268	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7269	if (RT_LIKELY(rc == VINF_SUCCESS))
7270	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7271	else
7272	{
7273	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7274	rc = VERR_EM_INTERPRETER;
7275	}
7276	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7277	return rc;
7278	}
7279
7280
7281	/**
7282	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7283	*/
7284	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7285	{
7286	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7287	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7288	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7289	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7290	AssertRCReturn(rc, rc);
7291
7292	PVM pVM = pVCpu->CTX_SUFF(pVM);
7293	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7294	rc = VBOXSTRICTRC_VAL(rc2);
7295	if (RT_LIKELY( rc == VINF_SUCCESS
7296	\|\| rc == VINF_EM_HALT))
7297	{
7298	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7299	AssertRCReturn(rc3, rc3);
7300
7301	if ( rc == VINF_EM_HALT
7302	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7303	{
7304	rc = VINF_SUCCESS;
7305	}
7306	}
7307	else
7308	{
7309	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7310	rc = VERR_EM_INTERPRETER;
7311	}
7312	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7313	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7314	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7315	return rc;
7316	}
7317
7318
7319	/**
7320	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7321	*/
7322	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7323	{
7324	/*
7325	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7326	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7327	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7328	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7329	*/
7330	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7331	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7332	}
7333
7334
7335	/**
7336	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7337	*/
7338	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7339	{
7340	/*
7341	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7342	* root operation. If we get there there is something funny going on.
7343	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7344	*/
7345	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7346	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7347	}
7348
7349
7350	/**
7351	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7352	*/
7353	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7354	{
7355	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7356	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7357	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7358	}
7359
7360
7361	/**
7362	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7363	*/
7364	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7365	{
7366	/*
7367	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7368	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7369	* See Intel spec. 25.3 "Other Causes of VM-exits".
7370	*/
7371	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7372	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7373	}
7374
7375
7376	/**
7377	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7378	* VM-exit.
7379	*/
7380	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7381	{
7382	/*
7383	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7384	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7385	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7386	*/
7387	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7388	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7389	}
7390
7391
7392	/**
7393	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7394	* VM-exit.
7395	*/
7396	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7397	{
7398	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7399	return VINF_EM_RESET;
7400	}
7401
7402
7403	/**
7404	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7405	*/
7406	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7407	{
7408	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7409	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7410	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7411	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7412	AssertRCReturn(rc, rc);
7413
7414	pMixedCtx->rip++;
7415	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7416	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7417	rc = VINF_SUCCESS;
7418	else
7419	rc = VINF_EM_HALT;
7420
7421	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7422	return rc;
7423	}
7424
7425
7426	/**
7427	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7428	*/
7429	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7430	{
7431	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7432	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7433	return VINF_SUCCESS;
7434	}
7435
7436
7437	/**
7438	* VM-exit handler for expiry of the VMX preemption timer.
7439	*/
7440	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7441	{
7442	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7443
7444	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7445	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7446
7447	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7448	PVM pVM = pVCpu->CTX_SUFF(pVM);
7449	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7450	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7451	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7452	}
7453
7454
7455	/**
7456	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7457	*/
7458	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7459	{
7460	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7461	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7462	/** @todo check if XSETBV is supported by the recompiler. */
7463	return VERR_EM_INTERPRETER;
7464	}
7465
7466
7467	/**
7468	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7469	*/
7470	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7471	{
7472	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7473	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7474	/** @todo implement EMInterpretInvpcid() */
7475	return VERR_EM_INTERPRETER;
7476	}
7477
7478
7479	/**
7480	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7481	* Error VM-exit.
7482	*/
7483	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7484	{
7485	uint32_t uIntrState;
7486	HMVMXHCUINTREG uHCReg;
7487	uint64_t u64Val;
7488	uint32_t u32Val;
7489
7490	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7491	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7492	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7493	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7494	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7495	AssertRCReturn(rc, rc);
7496
7497	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7498	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7499	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7500	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7501
7502	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7503	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7504	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7505	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7506	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7507	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7508	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7509	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7510	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7511	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7512	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7513	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7514
7515	PVM pVM = pVCpu->CTX_SUFF(pVM);
7516	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7517
7518	return VERR_VMX_INVALID_GUEST_STATE;
7519	}
7520
7521
7522	/**
7523	* VM-exit handler for VM-entry failure due to an MSR-load
7524	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7525	*/
7526	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7527	{
7528	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7529	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7530	}
7531
7532
7533	/**
7534	* VM-exit handler for VM-entry failure due to a machine-check event
7535	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7536	*/
7537	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7538	{
7539	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7540	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7541	}
7542
7543
7544	/**
7545	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7546	* theory.
7547	*/
7548	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7549	{
7550	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7551	return VERR_VMX_UNDEFINED_EXIT_CODE;
7552	}
7553
7554
7555	/**
7556	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7557	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7558	* Conditional VM-exit.
7559	*/
7560	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7561	{
7562	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7563	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7565	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7566	return VERR_EM_INTERPRETER;
7567	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7568	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7569	}
7570
7571
7572	/**
7573	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7574	*/
7575	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7576	{
7577	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7578	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7580	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7581	return VERR_EM_INTERPRETER;
7582	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7583	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7584	}
7585
7586
7587	/**
7588	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7589	*/
7590	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7591	{
7592	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7593	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7594	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7595	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7596	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7597	AssertRCReturn(rc, rc);
7598
7599	PVM pVM = pVCpu->CTX_SUFF(pVM);
7600	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7601	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7602	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7604
7605	if (RT_LIKELY(rc == VINF_SUCCESS))
7606	{
7607	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7608	Assert(pVmxTransient->cbInstr == 2);
7609	}
7610	return rc;
7611	}
7612
7613
7614	/**
7615	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7616	*/
7617	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7618	{
7619	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7620	PVM pVM = pVCpu->CTX_SUFF(pVM);
7621	int rc = VINF_SUCCESS;
7622
7623	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7624	if ( pVM->hm.s.fTPRPatchingActive
7625	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7626	{
7627	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7628	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7629	{
7630	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7631	AssertRC(rc);
7632	}
7633
7634	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7635	Assert(pVmxTransient->cbInstr == 2);
7636	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7637	return VINF_SUCCESS;
7638	}
7639
7640	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7641	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7642	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7643	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7644	AssertRCReturn(rc, rc);
7645	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7646
7647	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7648	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7650
7651	if (RT_LIKELY(rc == VINF_SUCCESS))
7652	{
7653	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7654
7655	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7656	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7657	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7658	{
7659	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7660	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7661	}
7662	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7663	{
7664	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7665	AssertRCReturn(rc, rc);
7666	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7667	}
7668	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7669	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7670
7671	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7672	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7673	{
7674	switch (pMixedCtx->ecx)
7675	{
7676	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7677	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7678	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7679	case MSR_K8_FS_BASE: /* no break */
7680	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7681	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7682	}
7683	}
7684	#ifdef VBOX_STRICT
7685	else
7686	{
7687	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7688	switch (pMixedCtx->ecx)
7689	{
7690	case MSR_IA32_SYSENTER_CS:
7691	case MSR_IA32_SYSENTER_EIP:
7692	case MSR_IA32_SYSENTER_ESP:
7693	case MSR_K8_FS_BASE:
7694	case MSR_K8_GS_BASE:
7695	{
7696	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7697	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7698	}
7699
7700	case MSR_K8_LSTAR:
7701	case MSR_K6_STAR:
7702	case MSR_K8_SF_MASK:
7703	case MSR_K8_TSC_AUX:
7704	case MSR_K8_KERNEL_GS_BASE:
7705	{
7706	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7707	pMixedCtx->ecx));
7708	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7709	}
7710	}
7711	}
7712	#endif /* VBOX_STRICT */
7713	}
7714	return rc;
7715	}
7716
7717
7718	/**
7719	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7720	*/
7721	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7722	{
7723	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7724	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7726	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7727	return VERR_EM_INTERPRETER;
7728	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7729	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7730	}
7731
7732
7733	/**
7734	* VM-exit handler for when the TPR value is lowered below the specified
7735	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7736	*/
7737	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7738	{
7739	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7740	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7741
7742	/*
7743	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7744	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7745	* resume guest execution.
7746	*/
7747	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7748	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7749	return VINF_SUCCESS;
7750	}
7751
7752
7753	/**
7754	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7755	* VM-exit.
7756	*
7757	* @retval VINF_SUCCESS when guest execution can continue.
7758	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7759	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7760	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7761	* recompiler.
7762	*/
7763	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7764	{
7765	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7766	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7767	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7768	AssertRCReturn(rc, rc);
7769
7770	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7771	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7772	PVM pVM = pVCpu->CTX_SUFF(pVM);
7773	switch (uAccessType)
7774	{
7775	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7776	{
7777	#if 0
7778	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7779	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7780	#else
7781	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7782	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7783	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7784	#endif
7785	AssertRCReturn(rc, rc);
7786
7787	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7788	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7789	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7790	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7791
7792	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7793	{
7794	case 0: /* CR0 */
7795	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7796	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7797	break;
7798	case 2: /* C2 **/
7799	/* Nothing to do here, CR2 it's not part of the VMCS. */
7800	break;
7801	case 3: /* CR3 */
7802	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7803	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7804	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7805	break;
7806	case 4: /* CR4 */
7807	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7808	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7809	break;
7810	case 8: /* CR8 */
7811	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7812	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7813	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7814	break;
7815	default:
7816	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7817	break;
7818	}
7819
7820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7821	break;
7822	}
7823
7824	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7825	{
7826	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7827	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7828	AssertRCReturn(rc, rc);
7829	Assert( !pVM->hm.s.fNestedPaging
7830	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7831	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7832
7833	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7834	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7835	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7836
7837	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7838	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7839	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7840	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7841	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7842	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7843	break;
7844	}
7845
7846	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7847	{
7848	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7849	AssertRCReturn(rc, rc);
7850	rc = EMInterpretCLTS(pVM, pVCpu);
7851	AssertRCReturn(rc, rc);
7852	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7854	Log(("CRX CLTS write rc=%d\n", rc));
7855	break;
7856	}
7857
7858	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7859	{
7860	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7861	AssertRCReturn(rc, rc);
7862	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7863	if (RT_LIKELY(rc == VINF_SUCCESS))
7864	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7865	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7866	Log(("CRX LMSW write rc=%d\n", rc));
7867	break;
7868	}
7869
7870	default:
7871	{
7872	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7873	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7874	}
7875	}
7876
7877	/* Validate possible error codes. */
7878	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7879	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7880	if (RT_SUCCESS(rc))
7881	{
7882	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7883	AssertRCReturn(rc2, rc2);
7884	}
7885
7886	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7887	return rc;
7888	}
7889
7890
7891	/**
7892	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7893	* VM-exit.
7894	*/
7895	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7896	{
7897	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7898	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7899
7900	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7901	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7902	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7903	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7904	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7905	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7906	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7907	AssertRCReturn(rc, rc);
7908	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7909
7910	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7911	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7912	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7913	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7914	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7915	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7916	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7917
7918	/* I/O operation lookup arrays. */
7919	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7920	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7921
7922	const uint32_t cbSize = s_aIOSize[uIOWidth];
7923	const uint32_t cbInstr = pVmxTransient->cbInstr;
7924	PVM pVM = pVCpu->CTX_SUFF(pVM);
7925	if (fIOString)
7926	{
7927	/* INS/OUTS - I/O String instruction. */
7928	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7929	/** @todo for now manually disassemble later optimize by getting the fields from
7930	* the VMCS. */
7931	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7932	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7933	* segment prefix info. */
7934	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7935	if (RT_SUCCESS(rc))
7936	{
7937	if (fIOWrite)
7938	{
7939	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7940	(DISCPUMODE)pDis->uAddrMode, cbSize);
7941	rc = VBOXSTRICTRC_VAL(rc2);
7942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7943	}
7944	else
7945	{
7946	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7947	(DISCPUMODE)pDis->uAddrMode, cbSize);
7948	rc = VBOXSTRICTRC_VAL(rc2);
7949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7950	}
7951	}
7952	else
7953	{
7954	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7955	rc = VINF_EM_RAW_EMULATE_INSTR;
7956	}
7957	}
7958	else
7959	{
7960	/* IN/OUT - I/O instruction. */
7961	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7962	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7963	if (fIOWrite)
7964	{
7965	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7966	rc = VBOXSTRICTRC_VAL(rc2);
7967	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7968	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7969	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7970	}
7971	else
7972	{
7973	uint32_t u32Result = 0;
7974	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7975	rc = VBOXSTRICTRC_VAL(rc2);
7976	if (IOM_SUCCESS(rc))
7977	{
7978	/* Save result of I/O IN instr. in AL/AX/EAX. */
7979	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7980	}
7981	else if (rc == VINF_IOM_R3_IOPORT_READ)
7982	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7983	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7984	}
7985	}
7986
7987	if (IOM_SUCCESS(rc))
7988	{
7989	pMixedCtx->rip += cbInstr;
7990	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7991	if (RT_LIKELY(rc == VINF_SUCCESS))
7992	{
7993	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7994	AssertRCReturn(rc, rc);
7995
7996	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7997	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7998	{
7999	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8000	for (unsigned i = 0; i < 4; i++)
8001	{
8002	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8003	if ( ( uIOPort >= pMixedCtx->dr[i]
8004	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8005	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8006	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8007	{
8008	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8009	uint64_t uDR6 = ASMGetDR6();
8010
8011	/* Clear all breakpoint status flags and set the one we just hit. */
8012	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8013	uDR6 \|= (uint64_t)RT_BIT(i);
8014
8015	/*
8016	* Note: AMD64 Architecture Programmer's Manual 13.1:
8017	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8018	* be cleared by software after the contents have been read.
8019	*/
8020	ASMSetDR6(uDR6);
8021
8022	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8023	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8024
8025	/* Paranoia. */
8026	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8027	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8028	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8029
8030	/* Resync DR7 */
8031	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8032	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8033
8034	/* Set #DB to be injected into the VM and continue guest execution. */
8035	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8036	break;
8037	}
8038	}
8039	}
8040	}
8041	}
8042
8043	#ifdef DEBUG
8044	if (rc == VINF_IOM_R3_IOPORT_READ)
8045	Assert(!fIOWrite);
8046	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8047	Assert(fIOWrite);
8048	else
8049	{
8050	AssertMsg( RT_FAILURE(rc)
8051	\|\| rc == VINF_SUCCESS
8052	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8053	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8054	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8055	}
8056	#endif
8057
8058	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8059	return rc;
8060	}
8061
8062
8063	/**
8064	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8065	* VM-exit.
8066	*/
8067	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8068	{
8069	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8070
8071	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8072	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8073	AssertRCReturn(rc, rc);
8074	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8075	{
8076	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8077	AssertRCReturn(rc, rc);
8078	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8079	{
8080	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8081	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8082	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8083	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8084	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8085	{
8086	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8087	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8088
8089	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8090	Assert(!pVCpu->hm.s.Event.fPending);
8091	pVCpu->hm.s.Event.fPending = true;
8092	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8093	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8094	AssertRCReturn(rc, rc);
8095	if (fErrorCodeValid)
8096	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8097	else
8098	pVCpu->hm.s.Event.u32ErrCode = 0;
8099	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8100	&& uVector == X86_XCPT_PF)
8101	{
8102	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8103	}
8104	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8105	}
8106	}
8107	}
8108	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8109	* emulation. */
8110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8111	return VERR_EM_INTERPRETER;
8112	}
8113
8114
8115	/**
8116	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8117	*/
8118	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8119	{
8120	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8121	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8122	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8123	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8124	AssertRCReturn(rc, rc);
8125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8126	return VINF_EM_DBG_STOP;
8127	}
8128
8129
8130	/**
8131	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8132	*/
8133	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8134	{
8135	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8136
8137	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8138	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8139	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8140	return VINF_SUCCESS;
8141	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8142	return rc;
8143
8144	#if 0
8145	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8146	* just sync the whole thing. */
8147	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8148	#else
8149	/* Aggressive state sync. for now. */
8150	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8151	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8152	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8153	#endif
8154	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8155	AssertRCReturn(rc, rc);
8156
8157	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8158	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8159	switch (uAccessType)
8160	{
8161	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8162	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8163	{
8164	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8165	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8166	{
8167	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8168	}
8169
8170	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8171	GCPhys &= PAGE_BASE_GC_MASK;
8172	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8173	PVM pVM = pVCpu->CTX_SUFF(pVM);
8174	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8175	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8176
8177	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8178	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8179	CPUMCTX2CORE(pMixedCtx), GCPhys);
8180	rc = VBOXSTRICTRC_VAL(rc2);
8181	Log(("ApicAccess rc=%d\n", rc));
8182	if ( rc == VINF_SUCCESS
8183	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8184	\|\| rc == VERR_PAGE_NOT_PRESENT)
8185	{
8186	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8187	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8188	rc = VINF_SUCCESS;
8189	}
8190	break;
8191	}
8192
8193	default:
8194	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8195	rc = VINF_EM_RAW_EMULATE_INSTR;
8196	break;
8197	}
8198
8199	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8200	return rc;
8201	}
8202
8203
8204	/**
8205	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8206	* VM-exit.
8207	*/
8208	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8209	{
8210	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8211
8212	/* We should -not- get this VM-exit if the guest is debugging. */
8213	if (CPUMIsGuestDebugStateActive(pVCpu))
8214	{
8215	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8216	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8217	}
8218
8219	int rc = VERR_INTERNAL_ERROR_5;
8220	if ( !DBGFIsStepping(pVCpu)
8221	&& !CPUMIsHyperDebugStateActive(pVCpu))
8222	{
8223	/* Don't intercept MOV DRx. */
8224	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8225	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8226	AssertRCReturn(rc, rc);
8227
8228	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8229	PVM pVM = pVCpu->CTX_SUFF(pVM);
8230	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8231	AssertRC(rc);
8232	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8233
8234	#ifdef VBOX_WITH_STATISTICS
8235	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8236	AssertRCReturn(rc, rc);
8237	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8238	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8239	else
8240	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8241	#endif
8242	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8243	return VINF_SUCCESS;
8244	}
8245
8246	/*
8247	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8248	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8249	*/
8250	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8251	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8252	AssertRCReturn(rc, rc);
8253
8254	PVM pVM = pVCpu->CTX_SUFF(pVM);
8255	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8256	{
8257	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8258	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8259	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8260	if (RT_SUCCESS(rc))
8261	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8263	}
8264	else
8265	{
8266	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8267	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8268	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8269	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8270	}
8271
8272	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8273	if (RT_SUCCESS(rc))
8274	{
8275	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8276	AssertRCReturn(rc2, rc2);
8277	}
8278	return rc;
8279	}
8280
8281
8282	/**
8283	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8284	* Conditional VM-exit.
8285	*/
8286	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8287	{
8288	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8289	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8290
8291	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8292	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8293	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8294	return VINF_SUCCESS;
8295	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8296	return rc;
8297
8298	RTGCPHYS GCPhys = 0;
8299	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8300
8301	#if 0
8302	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8303	#else
8304	/* Aggressive state sync. for now. */
8305	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8306	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8307	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8308	#endif
8309	AssertRCReturn(rc, rc);
8310
8311	/*
8312	* If we succeed, resume guest execution.
8313	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8314	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8315	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8316	* weird case. See @bugref{6043}.
8317	*/
8318	PVM pVM = pVCpu->CTX_SUFF(pVM);
8319	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8320	rc = VBOXSTRICTRC_VAL(rc2);
8321	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8322	if ( rc == VINF_SUCCESS
8323	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8324	\|\| rc == VERR_PAGE_NOT_PRESENT)
8325	{
8326	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8327	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8328	return VINF_SUCCESS;
8329	}
8330	return rc;
8331	}
8332
8333
8334	/**
8335	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8336	* VM-exit.
8337	*/
8338	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8339	{
8340	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8341	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8342
8343	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8344	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8345	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8346	return VINF_SUCCESS;
8347	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8348	return rc;
8349
8350	RTGCPHYS GCPhys = 0;
8351	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8352	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8353	#if 0
8354	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8355	#else
8356	/* Aggressive state sync. for now. */
8357	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8358	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8359	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8360	#endif
8361	AssertRCReturn(rc, rc);
8362
8363	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8364	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8365
8366	RTGCUINT uErrorCode = 0;
8367	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8368	uErrorCode \|= X86_TRAP_PF_ID;
8369	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8370	uErrorCode \|= X86_TRAP_PF_RW;
8371	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8372	uErrorCode \|= X86_TRAP_PF_P;
8373
8374	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8375
8376	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8377	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8378
8379	/* Handle the pagefault trap for the nested shadow table. */
8380	PVM pVM = pVCpu->CTX_SUFF(pVM);
8381	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8382	TRPMResetTrap(pVCpu);
8383
8384	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8385	if ( rc == VINF_SUCCESS
8386	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8387	\|\| rc == VERR_PAGE_NOT_PRESENT)
8388	{
8389	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8390	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8391	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8392	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8393	return VINF_SUCCESS;
8394	}
8395
8396	Log(("EPT return to ring-3 rc=%d\n"));
8397	return rc;
8398	}
8399
8400
8401	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8402	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8403	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8404	/**
8405	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8406	*/
8407	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8408	{
8409	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8410	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8411
8412	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8413	AssertRCReturn(rc, rc);
8414
8415	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8416	{
8417	/* Old-style FPU error reporting needs some extra work. */
8418	/** @todo don't fall back to the recompiler, but do it manually. */
8419	return VERR_EM_INTERPRETER;
8420	}
8421	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8422	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8423	return rc;
8424	}
8425
8426
8427	/**
8428	* VM-exit exception handler for #BP (Breakpoint exception).
8429	*/
8430	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8431	{
8432	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8433	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8434
8435	/** @todo Try optimize this by not saving the entire guest state unless
8436	* really needed. */
8437	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8438	AssertRCReturn(rc, rc);
8439
8440	PVM pVM = pVCpu->CTX_SUFF(pVM);
8441	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8442	if (rc == VINF_EM_RAW_GUEST_TRAP)
8443	{
8444	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8445	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8446	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8447	AssertRCReturn(rc, rc);
8448
8449	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8450	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8451	}
8452
8453	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8454	return rc;
8455	}
8456
8457
8458	/**
8459	* VM-exit exception handler for #DB (Debug exception).
8460	*/
8461	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8462	{
8463	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8464	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8465
8466	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8467	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8468	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8469	AssertRCReturn(rc, rc);
8470
8471	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8472	uint64_t uDR6 = X86_DR6_INIT_VAL;
8473	uDR6 \|= (pVmxTransient->uExitQualification
8474	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8475	PVM pVM = pVCpu->CTX_SUFF(pVM);
8476	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8477	if (rc == VINF_EM_RAW_GUEST_TRAP)
8478	{
8479	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8480	pMixedCtx->dr[6] = uDR6;
8481
8482	if (CPUMIsGuestDebugStateActive(pVCpu))
8483	ASMSetDR6(pMixedCtx->dr[6]);
8484
8485	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8486
8487	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8488	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8489
8490	/* Paranoia. */
8491	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8492	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8493	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8494
8495	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8496	AssertRCReturn(rc,rc);
8497
8498	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8499	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8500	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8501	AssertRCReturn(rc2, rc2);
8502	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8503	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8504	rc = VINF_SUCCESS;
8505	}
8506
8507	return rc;
8508	}
8509
8510
8511	/**
8512	* VM-exit exception handler for #NM (Device-not-available exception: floating
8513	* point exception).
8514	*/
8515	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8516	{
8517	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8518
8519	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8520	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8521	#endif
8522
8523	/* We require CR0 and EFER. EFER is always up-to-date. */
8524	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8525	AssertRCReturn(rc, rc);
8526
8527	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8528	PVM pVM = pVCpu->CTX_SUFF(pVM);
8529	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8530	if (rc == VINF_SUCCESS)
8531	{
8532	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8533	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8534	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8535	return VINF_SUCCESS;
8536	}
8537
8538	/* Forward #NM to the guest. */
8539	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8540	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8541	AssertRCReturn(rc, rc);
8542	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8543	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8544	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8545	return rc;
8546	}
8547
8548
8549	/**
8550	* VM-exit exception handler for #GP (General-protection exception).
8551	*
8552	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8553	*/
8554	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8555	{
8556	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8558
8559	int rc = VERR_INTERNAL_ERROR_5;
8560	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8561	{
8562	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8563	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8564	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8565	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8566	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8567	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8568	AssertRCReturn(rc, rc);
8569	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8570	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8571	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8572	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8573	return rc;
8574	#else
8575	/* We don't intercept #GP. */
8576	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8577	return VERR_VMX_UNEXPECTED_EXCEPTION;
8578	#endif
8579	}
8580
8581	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8582	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8583
8584	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8585	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8586	AssertRCReturn(rc, rc);
8587
8588	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8589	uint32_t cbOp = 0;
8590	PVM pVM = pVCpu->CTX_SUFF(pVM);
8591	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8592	if (RT_SUCCESS(rc))
8593	{
8594	rc = VINF_SUCCESS;
8595	Assert(cbOp == pDis->cbInstr);
8596	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8597	switch (pDis->pCurInstr->uOpcode)
8598	{
8599	case OP_CLI:
8600	pMixedCtx->eflags.Bits.u1IF = 0;
8601	pMixedCtx->rip += pDis->cbInstr;
8602	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8603	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8604	break;
8605
8606	case OP_STI:
8607	pMixedCtx->eflags.Bits.u1IF = 1;
8608	pMixedCtx->rip += pDis->cbInstr;
8609	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8610	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8611	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8612	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8613	break;
8614
8615	case OP_HLT:
8616	rc = VINF_EM_HALT;
8617	pMixedCtx->rip += pDis->cbInstr;
8618	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8620	break;
8621
8622	case OP_POPF:
8623	{
8624	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8625	uint32_t cbParm = 0;
8626	uint32_t uMask = 0;
8627	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8628	{
8629	cbParm = 4;
8630	uMask = 0xffffffff;
8631	}
8632	else
8633	{
8634	cbParm = 2;
8635	uMask = 0xffff;
8636	}
8637
8638	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8639	RTGCPTR GCPtrStack = 0;
8640	X86EFLAGS uEflags;
8641	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8642	&GCPtrStack);
8643	if (RT_SUCCESS(rc))
8644	{
8645	Assert(sizeof(uEflags.u32) >= cbParm);
8646	uEflags.u32 = 0;
8647	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8648	}
8649	if (RT_FAILURE(rc))
8650	{
8651	rc = VERR_EM_INTERPRETER;
8652	break;
8653	}
8654	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8655	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8656	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8657	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8658	pMixedCtx->eflags.Bits.u1RF = 0;
8659	pMixedCtx->esp += cbParm;
8660	pMixedCtx->esp &= uMask;
8661	pMixedCtx->rip += pDis->cbInstr;
8662	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8663	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8664	break;
8665	}
8666
8667	case OP_PUSHF:
8668	{
8669	uint32_t cbParm = 0;
8670	uint32_t uMask = 0;
8671	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8672	{
8673	cbParm = 4;
8674	uMask = 0xffffffff;
8675	}
8676	else
8677	{
8678	cbParm = 2;
8679	uMask = 0xffff;
8680	}
8681
8682	/* Get the stack pointer & push the contents of eflags onto the stack. */
8683	RTGCPTR GCPtrStack = 0;
8684	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8685	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8686	if (RT_FAILURE(rc))
8687	{
8688	rc = VERR_EM_INTERPRETER;
8689	break;
8690	}
8691	X86EFLAGS uEflags;
8692	uEflags = pMixedCtx->eflags;
8693	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8694	uEflags.Bits.u1RF = 0;
8695	uEflags.Bits.u1VM = 0;
8696
8697	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8698	if (RT_FAILURE(rc))
8699	{
8700	rc = VERR_EM_INTERPRETER;
8701	break;
8702	}
8703	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8704	pMixedCtx->esp -= cbParm;
8705	pMixedCtx->esp &= uMask;
8706	pMixedCtx->rip += pDis->cbInstr;
8707	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8709	break;
8710	}
8711
8712	case OP_IRET:
8713	{
8714	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8715	* instruction reference. */
8716	RTGCPTR GCPtrStack = 0;
8717	uint32_t uMask = 0xffff;
8718	uint16_t aIretFrame[3];
8719	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8720	{
8721	rc = VERR_EM_INTERPRETER;
8722	break;
8723	}
8724	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8725	&GCPtrStack);
8726	if (RT_SUCCESS(rc))
8727	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8728	if (RT_FAILURE(rc))
8729	{
8730	rc = VERR_EM_INTERPRETER;
8731	break;
8732	}
8733	pMixedCtx->eip = 0;
8734	pMixedCtx->ip = aIretFrame[0];
8735	pMixedCtx->cs.Sel = aIretFrame[1];
8736	pMixedCtx->cs.ValidSel = aIretFrame[1];
8737	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8738	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8739	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8740	pMixedCtx->sp += sizeof(aIretFrame);
8741	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8742	\| HM_CHANGED_GUEST_RFLAGS;
8743	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8744	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8745	break;
8746	}
8747
8748	case OP_INT:
8749	{
8750	uint16_t uVector = pDis->Param1.uValue & 0xff;
8751	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8752	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8753	break;
8754	}
8755
8756	case OP_INTO:
8757	{
8758	if (pMixedCtx->eflags.Bits.u1OF)
8759	{
8760	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8761	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8762	}
8763	break;
8764	}
8765
8766	default:
8767	{
8768	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8769	EMCODETYPE_SUPERVISOR);
8770	rc = VBOXSTRICTRC_VAL(rc2);
8771	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8772	Log(("#GP rc=%Rrc\n", rc));
8773	break;
8774	}
8775	}
8776	}
8777	else
8778	rc = VERR_EM_INTERPRETER;
8779
8780	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8781	("#GP Unexpected rc=%Rrc\n", rc));
8782	return rc;
8783	}
8784
8785
8786	/**
8787	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8788	* the exception reported in the VMX transient structure back into the VM.
8789	*
8790	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8791	* VMX transient structure to be up-to-date.
8792	*/
8793	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8794	{
8795	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8796
8797	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8798	hmR0VmxCheckExitDueToEventDelivery(). */
8799	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8800	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8801	AssertRCReturn(rc, rc);
8802	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8803	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8804	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8805	return VINF_SUCCESS;
8806	}
8807
8808
8809	/**
8810	* VM-exit exception handler for #PF (Page-fault exception).
8811	*/
8812	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8813	{
8814	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8815	PVM pVM = pVCpu->CTX_SUFF(pVM);
8816	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8817	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8818	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8819	AssertRCReturn(rc, rc);
8820
8821	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8822	if (pVM->hm.s.fNestedPaging)
8823	{
8824	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8825	{
8826	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8827	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8828	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8829	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8830	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8831	}
8832	else
8833	{
8834	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8835	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8836	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8837	Log(("Pending #DF due to vectoring #PF. NP\n"));
8838	}
8839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8840	return rc;
8841	}
8842	#else
8843	Assert(!pVM->hm.s.fNestedPaging);
8844	#endif
8845
8846	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8847	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8848	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8849	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8850	AssertRCReturn(rc, rc);
8851	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8852	if ( pVM->hm.s.fTRPPatchingAllowed
8853	&& pVM->hm.s.pGuestPatchMem
8854	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8855	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8856	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8857	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8858	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8859	{
8860	RTGCPHYS GCPhys;
8861	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8862	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8863	if ( rc == VINF_SUCCESS
8864	&& GCPhys == GCPhysApicBase)
8865	{
8866	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8867	AssertRCReturn(rc, rc);
8868
8869	/* Only attempt to patch the instruction once. */
8870	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8871	if (!pPatch)
8872	return VINF_EM_HM_PATCH_TPR_INSTR;
8873	}
8874	}
8875	#endif
8876
8877	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8878	AssertRCReturn(rc, rc);
8879
8880	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8881	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8882
8883	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8884	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8885	(RTGCPTR)pVmxTransient->uExitQualification);
8886
8887	Log(("#PF: rc=%Rrc\n", rc));
8888	if (rc == VINF_SUCCESS)
8889	{
8890	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8891	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8892	* memory? We don't update the whole state here... */
8893	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8894	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8895	TRPMResetTrap(pVCpu);
8896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8897	return rc;
8898	}
8899	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8900	{
8901	if (!pVmxTransient->fVectoringPF)
8902	{
8903	/* It's a guest page fault and needs to be reflected to the guest. */
8904	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8905	TRPMResetTrap(pVCpu);
8906	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8907	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8908	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8909	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8910	}
8911	else
8912	{
8913	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8914	TRPMResetTrap(pVCpu);
8915	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8916	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8917	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8918	}
8919
8920	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8921	return VINF_SUCCESS;
8922	}
8923
8924	TRPMResetTrap(pVCpu);
8925	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8926	return rc;
8927	}
8928
8929

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45849

Download in other formats: