HMVMXR0.cpp@ 45852

Last change on this file since 45852 was 45852, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: wrap to 130 columns.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 375.6 KB

Line
1	/* $Id: HMVMXR0.cpp 45852 2013-04-30 18:55:55Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** Updated-guest-state flags. */
73	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
74	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
75	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
76	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
77	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
78	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
79	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
80	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
81	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
82	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
83	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
84	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
85	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
86	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
87	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
90	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
91	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
92	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
93	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
94	\| HMVMX_UPDATED_GUEST_RSP \
95	\| HMVMX_UPDATED_GUEST_RFLAGS \
96	\| HMVMX_UPDATED_GUEST_CR0 \
97	\| HMVMX_UPDATED_GUEST_CR3 \
98	\| HMVMX_UPDATED_GUEST_CR4 \
99	\| HMVMX_UPDATED_GUEST_GDTR \
100	\| HMVMX_UPDATED_GUEST_IDTR \
101	\| HMVMX_UPDATED_GUEST_LDTR \
102	\| HMVMX_UPDATED_GUEST_TR \
103	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
104	\| HMVMX_UPDATED_GUEST_DEBUG \
105	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
106	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
112	\| HMVMX_UPDATED_GUEST_APIC_STATE)
113
114	/**
115	* Flags to skip redundant reads of some common VMCS fields that are not part of
116	* the guest-CPU state but are in the transient structure.
117	*/
118	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
119	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
121	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
124
125	/**
126	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
127	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
128	* we have Nested Paging support.
129	*/
130	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
131	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
132	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
133	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
134	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
135	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
136	\| RT_BIT(X86_XCPT_XF))
137
138	/**
139	* Exception bitmap mask for all contributory exceptions.
140	*/
141	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
142	\| RT_BIT(X86_XCPT_DE))
143
144	/** Maximum VM-instruction error number. */
145	#define HMVMX_INSTR_ERROR_MAX 28
146
147
148	/*******************************************************************************
149	* Structures and Typedefs *
150	*******************************************************************************/
151	/**
152	* A state structure for holding miscellaneous information across
153	* VMX non-root operation and restored after the transition.
154	*/
155	typedef struct VMXTRANSIENT
156	{
157	/** The host's rflags/eflags. */
158	RTCCUINTREG uEFlags;
159	#if HC_ARCH_BITS == 32
160	uint32_t u32Alignment0;
161	#endif
162	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
163	uint64_t u64LStarMsr;
164	/** The guest's TPR value used for TPR shadowing. */
165	uint8_t u8GuestTpr;
166	/** Alignment. */
167	uint8_t abAlignment0[6];
168
169	/** The basic VM-exit reason. */
170	uint16_t uExitReason;
171	/** Alignment. */
172	uint16_t u16Alignment0;
173	/** The VM-exit interruption error code. */
174	uint32_t uExitIntrErrorCode;
175	/** The VM-exit exit qualification. */
176	RTGCUINTPTR uExitQualification;
177	#if GC_ARCH_BITS == 32
178	/** Alignment. */
179	uint32_t u32Alignment1;
180	#endif
181
182	/** The VM-exit interruption-information field. */
183	uint32_t uExitIntrInfo;
184	/** The VM-exit instruction-length field. */
185	uint32_t cbInstr;
186	/** Whether the VM-entry failed or not. */
187	bool fVMEntryFailed;
188	/** Alignment. */
189	uint8_t abAlignment1[5];
190
191	/** The VM-entry interruption-information field. */
192	uint32_t uEntryIntrInfo;
193	/** The VM-entry exception error code field. */
194	uint32_t uEntryXcptErrorCode;
195	/** The VM-entry instruction length field. */
196	uint32_t cbEntryInstr;
197
198	/** IDT-vectoring information field. */
199	uint32_t uIdtVectoringInfo;
200	/** IDT-vectoring error code. */
201	uint32_t uIdtVectoringErrorCode;
202
203	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
204	uint32_t fVmcsFieldsRead;
205	/** Whether TSC-offsetting should be setup before VM-entry. */
206	bool fUpdateTscOffsettingAndPreemptTimer;
207	/** Whether the VM-exit was caused by a page-fault during delivery of a
208	* contributary exception or a page-fault. */
209	bool fVectoringPF;
210	} VMXTRANSIENT, *PVMXTRANSIENT;
211	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
212	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
213	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
214
215
216	/**
217	* MSR-bitmap read permissions.
218	*/
219	typedef enum VMXMSREXITREAD
220	{
221	/** Reading this MSR causes a VM-exit. */
222	VMXMSREXIT_INTERCEPT_READ = 0xb,
223	/** Reading this MSR does not cause a VM-exit. */
224	VMXMSREXIT_PASSTHRU_READ
225	} VMXMSREXITREAD;
226
227	/**
228	* MSR-bitmap write permissions.
229	*/
230	typedef enum VMXMSREXITWRITE
231	{
232	/** Writing to this MSR causes a VM-exit. */
233	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
234	/** Writing to this MSR does not cause a VM-exit. */
235	VMXMSREXIT_PASSTHRU_WRITE
236	} VMXMSREXITWRITE;
237
238
239	/*******************************************************************************
240	* Internal Functions *
241	*******************************************************************************/
242	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
243	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
244	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
245	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
246	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
247	#endif
248	#ifndef HMVMX_USE_FUNCTION_TABLE
249	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
250	#endif
251
252	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296
297	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304
305
306	/*******************************************************************************
307	* Global Variables *
308	*******************************************************************************/
309	#ifdef HMVMX_USE_FUNCTION_TABLE
310	/**
311	* VM-exit handler.
312	*
313	* @returns VBox status code.
314	* @param pVCpu Pointer to the VMCPU.
315	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
316	* out-of-sync. Make sure to update the required
317	* fields before using them.
318	* @param pVmxTransient Pointer to the VMX-transient structure.
319	*/
320	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
321	/** Pointer to VM-exit handler. */
322	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
323
324	/**
325	* VMX_EXIT dispatch table.
326	*/
327	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
328	{
329	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
330	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
331	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
332	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
333	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
334	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
335	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
336	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
337	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
338	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
339	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
340	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
341	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
342	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
343	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
344	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
345	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
346	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
347	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
348	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
349	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
350	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
351	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
352	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
353	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
354	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
355	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
356	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
357	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
358	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
359	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
360	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
361	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
362	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
363	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
364	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
365	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
366	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
367	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
368	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
369	/* 40 UNDEFINED */ hmR0VmxExitPause,
370	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
371	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
372	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
373	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
374	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
376	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
377	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
378	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
379	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
380	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
381	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
382	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
383	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
384	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
385	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
386	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
387	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
388	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
389	};
390	#endif /* HMVMX_USE_FUNCTION_TABLE */
391
392	#ifdef VBOX_STRICT
393	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
394	{
395	/* 0 */ "(Not Used)",
396	/* 1 */ "VMCALL executed in VMX root operation.",
397	/* 2 */ "VMCLEAR with invalid physical address.",
398	/* 3 */ "VMCLEAR with VMXON pointer.",
399	/* 4 */ "VMLAUNCH with non-clear VMCS.",
400	/* 5 */ "VMRESUME with non-launched VMCS.",
401	/* 6 */ "VMRESUME after VMXOFF",
402	/* 7 */ "VM entry with invalid control fields.",
403	/* 8 */ "VM entry with invalid host state fields.",
404	/* 9 */ "VMPTRLD with invalid physical address.",
405	/* 10 */ "VMPTRLD with VMXON pointer.",
406	/* 11 */ "VMPTRLD with incorrect revision identifier.",
407	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
408	/* 13 */ "VMWRITE to read-only VMCS component.",
409	/* 14 */ "(Not Used)",
410	/* 15 */ "VMXON executed in VMX root operation.",
411	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
412	/* 17 */ "VM entry with non-launched executing VMCS.",
413	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
414	/* 19 */ "VMCALL with non-clear VMCS.",
415	/* 20 */ "VMCALL with invalid VM-exit control fields.",
416	/* 21 */ "(Not Used)",
417	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
418	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
419	/* 24 */ "VMCALL with invalid SMM-monitor features.",
420	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
421	/* 26 */ "VM entry with events blocked by MOV SS.",
422	/* 27 */ "(Not Used)",
423	/* 28 */ "Invalid operand to INVEPT/INVVPID."
424	};
425	#endif /* VBOX_STRICT */
426
427
428
429	/**
430	* Updates the VM's last error record. If there was a VMX instruction error,
431	* reads the error data from the VMCS and updates VCPU's last error record as
432	* well.
433	*
434	* @param pVM Pointer to the VM.
435	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
436	* VERR_VMX_UNABLE_TO_START_VM or
437	* VERR_VMX_INVALID_VMCS_FIELD).
438	* @param rc The error code.
439	*/
440	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
441	{
442	AssertPtr(pVM);
443	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
444	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
445	{
446	AssertPtrReturnVoid(pVCpu);
447	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
448	}
449	pVM->hm.s.lLastError = rc;
450	}
451
452
453	/**
454	* Reads the VM-entry interruption-information field from the VMCS into the VMX
455	* transient structure.
456	*
457	* @returns VBox status code.
458	* @param pVmxTransient Pointer to the VMX transient structure.
459	*
460	* @remarks No-long-jump zone!!!
461	*/
462	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
463	{
464	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
465	AssertRCReturn(rc, rc);
466	return VINF_SUCCESS;
467	}
468
469
470	/**
471	* Reads the VM-entry exception error code field from the VMCS into
472	* the VMX transient structure.
473	*
474	* @returns VBox status code.
475	* @param pVmxTransient Pointer to the VMX transient structure.
476	*
477	* @remarks No-long-jump zone!!!
478	*/
479	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
480	{
481	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
482	AssertRCReturn(rc, rc);
483	return VINF_SUCCESS;
484	}
485
486
487	/**
488	* Reads the VM-entry exception error code field from the VMCS into
489	* the VMX transient structure.
490	*
491	* @returns VBox status code.
492	* @param pVCpu Pointer to the VMCPU.
493	* @param pVmxTransient Pointer to the VMX transient structure.
494	*
495	* @remarks No-long-jump zone!!!
496	*/
497	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
498	{
499	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
500	AssertRCReturn(rc, rc);
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption-information field from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit interruption error code from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the VM-exit instruction length field from the VMCS into the VMX
547	* transient structure.
548	*
549	* @returns VBox status code.
550	* @param pVCpu Pointer to the VMCPU.
551	* @param pVmxTransient Pointer to the VMX transient structure.
552	*/
553	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
554	{
555	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
556	{
557	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
558	AssertRCReturn(rc, rc);
559	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
560	}
561	return VINF_SUCCESS;
562	}
563
564
565	/**
566	* Reads the exit qualification from the VMCS into the VMX transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVCpu Pointer to the VMCPU.
570	* @param pVmxTransient Pointer to the VMX transient structure.
571	*/
572	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
573	{
574	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
575	{
576	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
577	AssertRCReturn(rc, rc);
578	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
579	}
580	return VINF_SUCCESS;
581	}
582
583
584	/**
585	* Reads the IDT-vectoring information field from the VMCS into the VMX
586	* transient structure.
587	*
588	* @returns VBox status code.
589	* @param pVmxTransient Pointer to the VMX transient structure.
590	*
591	* @remarks No-long-jump zone!!!
592	*/
593	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
594	{
595	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
596	{
597	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
598	AssertRCReturn(rc, rc);
599	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
600	}
601	return VINF_SUCCESS;
602	}
603
604
605	/**
606	* Reads the IDT-vectoring error code from the VMCS into the VMX
607	* transient structure.
608	*
609	* @returns VBox status code.
610	* @param pVmxTransient Pointer to the VMX transient structure.
611	*/
612	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
613	{
614	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
615	{
616	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
617	AssertRCReturn(rc, rc);
618	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
619	}
620	return VINF_SUCCESS;
621	}
622
623
624	/**
625	* Enters VMX root mode operation on the current CPU.
626	*
627	* @returns VBox status code.
628	* @param pVM Pointer to the VM (optional, can be NULL, after
629	* a resume).
630	* @param HCPhysCpuPage Physical address of the VMXON region.
631	* @param pvCpuPage Pointer to the VMXON region.
632	*/
633	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
634	{
635	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
636	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
637	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
638
639	if (pVM)
640	{
641	/* Write the VMCS revision dword to the VMXON region. */
642	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
643	}
644
645	/* Enable the VMX bit in CR4 if necessary. */
646	RTCCUINTREG uCr4 = ASMGetCR4();
647	if (!(uCr4 & X86_CR4_VMXE))
648	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
649
650	/* Enter VMX root mode. */
651	int rc = VMXEnable(HCPhysCpuPage);
652	if (RT_FAILURE(rc))
653	ASMSetCR4(uCr4);
654
655	return rc;
656	}
657
658
659	/**
660	* Exits VMX root mode operation on the current CPU.
661	*
662	* @returns VBox status code.
663	*/
664	static int hmR0VmxLeaveRootMode(void)
665	{
666	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
667
668	/* If we're for some reason not in VMX root mode, then don't leave it. */
669	if (ASMGetCR4() & X86_CR4_VMXE)
670	{
671	/* Exit VMX root mode and clear the VMX bit in CR4 */
672	VMXDisable();
673	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
674	return VINF_SUCCESS;
675	}
676
677	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
678	}
679
680
681	/**
682	* Allocates and maps one physically contiguous page. The allocated page is
683	* zero'd out. (Used by various VT-x structures).
684	*
685	* @returns IPRT status code.
686	* @param pMemObj Pointer to the ring-0 memory object.
687	* @param ppVirt Where to store the virtual address of the
688	* allocation.
689	* @param pPhys Where to store the physical address of the
690	* allocation.
691	*/
692	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
693	{
694	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
695	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
696	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
697
698	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
699	if (RT_FAILURE(rc))
700	return rc;
701	ppVirt = RTR0MemObjAddress(pMemObj);
702	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
703	ASMMemZero32(*ppVirt, PAGE_SIZE);
704	return VINF_SUCCESS;
705	}
706
707
708	/**
709	* Frees and unmaps an allocated physical page.
710	*
711	* @param pMemObj Pointer to the ring-0 memory object.
712	* @param ppVirt Where to re-initialize the virtual address of
713	* allocation as 0.
714	* @param pHCPhys Where to re-initialize the physical address of the
715	* allocation as 0.
716	*/
717	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
718	{
719	AssertPtr(pMemObj);
720	AssertPtr(ppVirt);
721	AssertPtr(pHCPhys);
722	if (*pMemObj != NIL_RTR0MEMOBJ)
723	{
724	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
725	AssertRC(rc);
726	*pMemObj = NIL_RTR0MEMOBJ;
727	*ppVirt = 0;
728	*pHCPhys = 0;
729	}
730	}
731
732
733	/**
734	* Worker function to free VT-x related structures.
735	*
736	* @returns IPRT status code.
737	* @param pVM Pointer to the VM.
738	*/
739	static void hmR0VmxStructsFree(PVM pVM)
740	{
741	for (VMCPUID i = 0; i < pVM->cCpus; i++)
742	{
743	PVMCPU pVCpu = &pVM->aCpus[i];
744	AssertPtr(pVCpu);
745
746	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
747	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
748	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
749	#endif
750
751	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
752	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
753
754	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
755	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
756	}
757
758	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
759	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
760	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
761	#endif
762	}
763
764
765	/**
766	* Worker function to allocate VT-x related VM structures.
767	*
768	* @returns IPRT status code.
769	* @param pVM Pointer to the VM.
770	*/
771	static int hmR0VmxStructsAlloc(PVM pVM)
772	{
773	/*
774	* Initialize members up-front so we can cleanup properly on allocation failure.
775	*/
776	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
777	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
778	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
779	pVM->hm.s.vmx.HCPhys##a_Name = 0;
780
781	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
782	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
783	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
784	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
785
786	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
787	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
788	#endif
789	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
790
791	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
792	for (VMCPUID i = 0; i < pVM->cCpus; i++)
793	{
794	PVMCPU pVCpu = &pVM->aCpus[i];
795	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
796	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
797	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
798	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
799	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
800	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
801	#endif
802	}
803	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
804	#undef VMXLOCAL_INIT_VM_MEMOBJ
805
806	/*
807	* Allocate all the VT-x structures.
808	*/
809	int rc = VINF_SUCCESS;
810	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
811	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
812	if (RT_FAILURE(rc))
813	goto cleanup;
814	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
815	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
816	#endif
817
818	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
820	{
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
822	&pVM->hm.s.vmx.HCPhysApicAccess);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	}
826
827	/*
828	* Initialize per-VCPU VT-x structures.
829	*/
830	for (VMCPUID i =0; i < pVM->cCpus; i++)
831	{
832	PVMCPU pVCpu = &pVM->aCpus[i];
833	AssertPtr(pVCpu);
834
835	/* Allocate the VM control structure (VMCS). */
836	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
837	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
838	if (RT_FAILURE(rc))
839	goto cleanup;
840
841	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
842	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
843	{
844	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
845	&pVCpu->hm.s.vmx.HCPhysVirtApic);
846	if (RT_FAILURE(rc))
847	goto cleanup;
848	}
849
850	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
851	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
852	{
853	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
854	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
855	if (RT_FAILURE(rc))
856	goto cleanup;
857	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
858	}
859
860	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
861	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
862	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
863	if (RT_FAILURE(rc))
864	goto cleanup;
865
866	/* Allocate the VM-exit MSR-load page for the host MSRs. */
867	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
868	if (RT_FAILURE(rc))
869	goto cleanup;
870	#endif
871	}
872
873	return VINF_SUCCESS;
874
875	cleanup:
876	hmR0VmxStructsFree(pVM);
877	return rc;
878	}
879
880
881	/**
882	* Does global VT-x initialization (called during module initialization).
883	*
884	* @returns VBox status code.
885	*/
886	VMMR0DECL(int) VMXR0GlobalInit(void)
887	{
888	#ifdef HMVMX_USE_FUNCTION_TABLE
889	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
890	# ifdef VBOX_STRICT
891	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
892	Assert(g_apfnVMExitHandlers[i]);
893	# endif
894	#endif
895	return VINF_SUCCESS;
896	}
897
898
899	/**
900	* Does global VT-x termination (called during module termination).
901	*/
902	VMMR0DECL(void) VMXR0GlobalTerm()
903	{
904	/* Nothing to do currently. */
905	}
906
907
908	/**
909	* Sets up and activates VT-x on the current CPU.
910	*
911	* @returns VBox status code.
912	* @param pCpu Pointer to the global CPU info struct.
913	* @param pVM Pointer to the VM (can be NULL after a host resume
914	* operation).
915	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
916	* fEnabledByHost is true).
917	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
918	* @a fEnabledByHost is true).
919	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
920	* enable VT-x/AMD-V on the host.
921	*/
922	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
923	{
924	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
925	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
926
927	if (!fEnabledByHost)
928	{
929	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
930	if (RT_FAILURE(rc))
931	return rc;
932	}
933
934	/*
935	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
936	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
937	* each time while reusing a VPID after hitting the MaxASID limit once.
938	*/
939	if ( pVM
940	&& pVM->hm.s.vmx.fVpid
941	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
942	{
943	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
944	pCpu->fFlushAsidBeforeUse = false;
945	}
946	else
947	pCpu->fFlushAsidBeforeUse = true;
948
949	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
950	++pCpu->cTlbFlushes;
951
952	return VINF_SUCCESS;
953	}
954
955
956	/**
957	* Deactivates VT-x on the current CPU.
958	*
959	* @returns VBox status code.
960	* @param pCpu Pointer to the global CPU info struct.
961	* @param pvCpuPage Pointer to the VMXON region.
962	* @param HCPhysCpuPage Physical address of the VMXON region.
963	*/
964	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
965	{
966	NOREF(pCpu);
967	NOREF(pvCpuPage);
968	NOREF(HCPhysCpuPage);
969
970	hmR0VmxLeaveRootMode();
971	return VINF_SUCCESS;
972	}
973
974
975	/**
976	* Sets the permission bits for the specified MSR in the MSR bitmap.
977	*
978	* @param pVCpu Pointer to the VMCPU.
979	* @param uMSR The MSR value.
980	* @param enmRead Whether reading this MSR causes a VM-exit.
981	* @param enmWrite Whether writing this MSR causes a VM-exit.
982	*/
983	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
984	{
985	int32_t iBit;
986	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
987
988	/*
989	* Layout:
990	* 0x000 - 0x3ff - Low MSR read bits
991	* 0x400 - 0x7ff - High MSR read bits
992	* 0x800 - 0xbff - Low MSR write bits
993	* 0xc00 - 0xfff - High MSR write bits
994	*/
995	if (uMsr <= 0x00001FFF)
996	iBit = uMsr;
997	else if ( uMsr >= 0xC0000000
998	&& uMsr <= 0xC0001FFF)
999	{
1000	iBit = (uMsr - 0xC0000000);
1001	pbMsrBitmap += 0x400;
1002	}
1003	else
1004	{
1005	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1006	return;
1007	}
1008
1009	Assert(iBit <= 0x1fff);
1010	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1011	ASMBitSet(pbMsrBitmap, iBit);
1012	else
1013	ASMBitClear(pbMsrBitmap, iBit);
1014
1015	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1016	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1017	else
1018	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1019	}
1020
1021
1022	/**
1023	* Flushes the TLB using EPT.
1024	*
1025	* @returns VBox status code.
1026	* @param pVM Pointer to the VM.
1027	* @param pVCpu Pointer to the VMCPU.
1028	* @param enmFlush Type of flush.
1029	*/
1030	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.fNestedPaging);
1034
1035	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1036
1037	uint64_t descriptor[2];
1038	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1039	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1040
1041	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1042	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1043	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1044	}
1045
1046
1047	/**
1048	* Flushes the TLB using VPID.
1049	*
1050	* @returns VBox status code.
1051	* @param pVM Pointer to the VM.
1052	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1053	* enmFlush).
1054	* @param enmFlush Type of flush.
1055	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1056	* on @a enmFlush).
1057	*/
1058	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1059	{
1060	AssertPtr(pVM);
1061	Assert(pVM->hm.s.vmx.fVpid);
1062
1063	uint64_t descriptor[2];
1064	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1065	{
1066	descriptor[0] = 0;
1067	descriptor[1] = 0;
1068	}
1069	else
1070	{
1071	AssertPtr(pVCpu);
1072	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1073	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1074	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1075	descriptor[1] = GCPtr;
1076	}
1077
1078	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1079	AssertMsg(rc == VINF_SUCCESS,
1080	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1081	if ( RT_SUCCESS(rc)
1082	&& pVCpu)
1083	{
1084	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1085	}
1086	}
1087
1088
1089	/**
1090	* Invalidates a guest page by guest virtual address. Only relevant for
1091	* EPT/VPID, otherwise there is nothing really to invalidate.
1092	*
1093	* @returns VBox status code.
1094	* @param pVM Pointer to the VM.
1095	* @param pVCpu Pointer to the VMCPU.
1096	* @param GCVirt Guest virtual address of the page to invalidate.
1097	*/
1098	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1099	{
1100	AssertPtr(pVM);
1101	AssertPtr(pVCpu);
1102	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1103
1104	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1105	if (!fFlushPending)
1106	{
1107	/*
1108	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1109	* See @bugref{6043} and @bugref{6177}.
1110	*
1111	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1112	* function maybe called in a loop with individual addresses.
1113	*/
1114	if (pVM->hm.s.vmx.fVpid)
1115	{
1116	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1117	{
1118	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1119	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1120	}
1121	else
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	}
1124	else if (pVM->hm.s.fNestedPaging)
1125	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1126	}
1127
1128	return VINF_SUCCESS;
1129	}
1130
1131
1132	/**
1133	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1134	* otherwise there is nothing really to invalidate.
1135	*
1136	* @returns VBox status code.
1137	* @param pVM Pointer to the VM.
1138	* @param pVCpu Pointer to the VMCPU.
1139	* @param GCPhys Guest physical address of the page to invalidate.
1140	*/
1141	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1142	{
1143	LogFlowFunc(("%RGp\n", GCPhys));
1144
1145	/*
1146	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1147	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1148	* This function might be called in a loop.
1149	*/
1150	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1151	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1152	return VINF_SUCCESS;
1153	}
1154
1155
1156	/**
1157	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1158	* case where neither EPT nor VPID is supported by the CPU.
1159	*
1160	* @param pVM Pointer to the VM.
1161	* @param pVCpu Pointer to the VMCPU.
1162	*
1163	* @remarks Called with interrupts disabled.
1164	*/
1165	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1166	{
1167	NOREF(pVM);
1168	AssertPtr(pVCpu);
1169	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1170	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1171
1172	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1173	AssertPtr(pCpu);
1174
1175	pVCpu->hm.s.TlbShootdown.cPages = 0;
1176	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1177	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1178	pVCpu->hm.s.fForceTLBFlush = false;
1179	return;
1180	}
1181
1182
1183	/**
1184	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1185	*
1186	* @param pVM Pointer to the VM.
1187	* @param pVCpu Pointer to the VMCPU.
1188	* @remarks All references to "ASID" in this function pertains to "VPID" in
1189	* Intel's nomenclature. The reason is, to avoid confusion in compare
1190	* statements since the host-CPU copies are named "ASID".
1191	*
1192	* @remarks Called with interrupts disabled.
1193	*/
1194	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1195	{
1196	AssertPtr(pVM);
1197	AssertPtr(pVCpu);
1198	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1199	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1200	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1201
1202	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1203	AssertPtr(pCpu);
1204
1205	/*
1206	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1207	* This can happen both for start & resume due to long jumps back to ring-3.
1208	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1209	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1210	*/
1211	bool fNewASID = false;
1212	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1213	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1214	{
1215	pVCpu->hm.s.fForceTLBFlush = true;
1216	fNewASID = true;
1217	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1218	}
1219
1220	/*
1221	* Check for explicit TLB shootdowns.
1222	*/
1223	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1224	{
1225	pVCpu->hm.s.fForceTLBFlush = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1227	}
1228
1229	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1230	if (pVCpu->hm.s.fForceTLBFlush)
1231	{
1232	if (fNewASID)
1233	{
1234	++pCpu->uCurrentAsid;
1235	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1236	{
1237	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1238	pCpu->cTlbFlushes++;
1239	pCpu->fFlushAsidBeforeUse = true;
1240	}
1241
1242	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1243	if (pCpu->fFlushAsidBeforeUse)
1244	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1245	}
1246	else
1247	{
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1249	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1250	else
1251	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1252	}
1253
1254	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1255	pVCpu->hm.s.fForceTLBFlush = false;
1256	}
1257	else
1258	{
1259	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1260	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1261	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1262	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1263
1264	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1265	* not be executed. See hmQueueInvlPage() where it is commented
1266	* out. Support individual entry flushing someday. */
1267	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1268	{
1269	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1270
1271	/*
1272	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1273	* as supported by the CPU.
1274	*/
1275	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1276	{
1277	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1278	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1279	}
1280	else
1281	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1282	}
1283	else
1284	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1285	}
1286	pVCpu->hm.s.TlbShootdown.cPages = 0;
1287	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1288
1289	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1290	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1291	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1292	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1293	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1294	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1295
1296	/* Update VMCS with the VPID. */
1297	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1298	AssertRC(rc);
1299	}
1300
1301
1302	/**
1303	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1304	*
1305	* @returns VBox status code.
1306	* @param pVM Pointer to the VM.
1307	* @param pVCpu Pointer to the VMCPU.
1308	*
1309	* @remarks Called with interrupts disabled.
1310	*/
1311	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1312	{
1313	AssertPtr(pVM);
1314	AssertPtr(pVCpu);
1315	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1316	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1317
1318	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1319	AssertPtr(pCpu);
1320
1321	/*
1322	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1323	* This can happen both for start & resume due to long jumps back to ring-3.
1324	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1325	*/
1326	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1327	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1328	{
1329	pVCpu->hm.s.fForceTLBFlush = true;
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1331	}
1332
1333	/* Check for explicit TLB shootdown flushes. */
1334	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1335	{
1336	pVCpu->hm.s.fForceTLBFlush = true;
1337	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1338	}
1339
1340	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1341	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1342
1343	if (pVCpu->hm.s.fForceTLBFlush)
1344	{
1345	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1346	pVCpu->hm.s.fForceTLBFlush = false;
1347	}
1348	else
1349	{
1350	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1351	* not be executed. See hmQueueInvlPage() where it is commented
1352	* out. Support individual entry flushing someday. */
1353	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1354	{
1355	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1356	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1357	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1358	}
1359	else
1360	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1361	}
1362
1363	pVCpu->hm.s.TlbShootdown.cPages = 0;
1364	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1365	}
1366
1367
1368	/**
1369	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1370	*
1371	* @returns VBox status code.
1372	* @param pVM Pointer to the VM.
1373	* @param pVCpu Pointer to the VMCPU.
1374	*
1375	* @remarks Called with interrupts disabled.
1376	*/
1377	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1378	{
1379	AssertPtr(pVM);
1380	AssertPtr(pVCpu);
1381	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1382	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1383
1384	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1385
1386	/*
1387	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1388	* This can happen both for start & resume due to long jumps back to ring-3.
1389	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1390	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1391	*/
1392	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1393	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1394	{
1395	pVCpu->hm.s.fForceTLBFlush = true;
1396	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1397	}
1398
1399	/* Check for explicit TLB shootdown flushes. */
1400	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1401	{
1402	/*
1403	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1404	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1405	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1406	*/
1407	pVCpu->hm.s.fForceTLBFlush = true;
1408	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1409	}
1410
1411	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1412	if (pVCpu->hm.s.fForceTLBFlush)
1413	{
1414	++pCpu->uCurrentAsid;
1415	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1416	{
1417	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1418	pCpu->fFlushAsidBeforeUse = true;
1419	pCpu->cTlbFlushes++;
1420	}
1421
1422	pVCpu->hm.s.fForceTLBFlush = false;
1423	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1424	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1425	if (pCpu->fFlushAsidBeforeUse)
1426	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1427	}
1428	else
1429	{
1430	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1431	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1432	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1433	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1434
1435	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1436	* not be executed. See hmQueueInvlPage() where it is commented
1437	* out. Support individual entry flushing someday. */
1438	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1439	{
1440	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1441	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1442	{
1443	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1444	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1445	}
1446	else
1447	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1448	}
1449	else
1450	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1451	}
1452
1453	pVCpu->hm.s.TlbShootdown.cPages = 0;
1454	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1455
1456	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1457	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1458	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1459	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1460	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1461	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1462
1463	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1464	AssertRC(rc);
1465	}
1466
1467
1468	/**
1469	* Flushes the guest TLB entry based on CPU capabilities.
1470	*
1471	* @param pVCpu Pointer to the VMCPU.
1472	*/
1473	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1474	{
1475	PVM pVM = pVCpu->CTX_SUFF(pVM);
1476	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1477	{
1478	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1479	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1480	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1481	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1482	default:
1483	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1484	break;
1485	}
1486	}
1487
1488
1489	/**
1490	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1491	* TLB entries from the host TLB before VM-entry.
1492	*
1493	* @returns VBox status code.
1494	* @param pVM Pointer to the VM.
1495	*/
1496	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1497	{
1498	/*
1499	* Determine optimal flush type for nested paging.
1500	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1501	* guest execution (see hmR3InitFinalizeR0()).
1502	*/
1503	if (pVM->hm.s.fNestedPaging)
1504	{
1505	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1506	{
1507	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1508	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1509	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1510	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1511	else
1512	{
1513	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1514	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1515	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1516	}
1517
1518	/* Make sure the write-back cacheable memory type for EPT is supported. */
1519	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1520	{
1521	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1522	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1523	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1524	}
1525	}
1526	else
1527	{
1528	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1529	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1530	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1531	}
1532	}
1533
1534	/*
1535	* Determine optimal flush type for VPID.
1536	*/
1537	if (pVM->hm.s.vmx.fVpid)
1538	{
1539	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1540	{
1541	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1542	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1543	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1544	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1545	else
1546	{
1547	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1549	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1551	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1552	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1553	pVM->hm.s.vmx.fVpid = false;
1554	}
1555	}
1556	else
1557	{
1558	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1559	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1560	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1561	pVM->hm.s.vmx.fVpid = false;
1562	}
1563	}
1564
1565	/*
1566	* Setup the handler for flushing tagged-TLBs.
1567	*/
1568	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1569	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1570	else if (pVM->hm.s.fNestedPaging)
1571	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1572	else if (pVM->hm.s.vmx.fVpid)
1573	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1574	else
1575	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1576	return VINF_SUCCESS;
1577	}
1578
1579
1580	/**
1581	* Sets up pin-based VM-execution controls in the VMCS.
1582	*
1583	* @returns VBox status code.
1584	* @param pVM Pointer to the VM.
1585	* @param pVCpu Pointer to the VMCPU.
1586	*/
1587	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1588	{
1589	AssertPtr(pVM);
1590	AssertPtr(pVCpu);
1591
1592	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1593	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1594
1595	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1596	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1597	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1598
1599	/* Enable the VMX preemption timer. */
1600	if (pVM->hm.s.vmx.fUsePreemptTimer)
1601	{
1602	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1603	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1604	}
1605
1606	if ((val & zap) != val)
1607	{
1608	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1609	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1610	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1611	}
1612
1613	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1614	AssertRCReturn(rc, rc);
1615
1616	/* Update VCPU with the currently set pin-based VM-execution controls. */
1617	pVCpu->hm.s.vmx.u32PinCtls = val;
1618	return rc;
1619	}
1620
1621
1622	/**
1623	* Sets up processor-based VM-execution controls in the VMCS.
1624	*
1625	* @returns VBox status code.
1626	* @param pVM Pointer to the VM.
1627	* @param pVMCPU Pointer to the VMCPU.
1628	*/
1629	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1630	{
1631	AssertPtr(pVM);
1632	AssertPtr(pVCpu);
1633
1634	int rc = VERR_INTERNAL_ERROR_5;
1635	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1636	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1637
1638	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1639	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1640	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1641	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1645
1646	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1647	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1648	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1649	{
1650	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1652	}
1653
1654	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1655	if (!pVM->hm.s.fNestedPaging)
1656	{
1657	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1658	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1659	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1660	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1661	}
1662
1663	/* Use TPR shadowing if supported by the CPU. */
1664	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1665	{
1666	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1667	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1668	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1669	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1670	AssertRCReturn(rc, rc);
1671
1672	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1673	/* CR8 writes causes a VM-exit based on TPR threshold. */
1674	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1675	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1676	}
1677	else
1678	{
1679	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1680	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1681	}
1682
1683	/* Use MSR-bitmaps if supported by the CPU. */
1684	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1685	{
1686	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1687
1688	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1689	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1690	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1691	AssertRCReturn(rc, rc);
1692
1693	/*
1694	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1695	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1696	*/
1697	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1698	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1699	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1700	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1701	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1702	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1703	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1704	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1705	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1706	}
1707
1708	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1710	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1711
1712	if ((val & zap) != val)
1713	{
1714	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1715	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1716	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1717	}
1718
1719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1720	AssertRCReturn(rc, rc);
1721
1722	/* Update VCPU with the currently set processor-based VM-execution controls. */
1723	pVCpu->hm.s.vmx.u32ProcCtls = val;
1724
1725	/*
1726	* Secondary processor-based VM-execution controls.
1727	*/
1728	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1729	{
1730	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1731	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1732
1733	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1734	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1735
1736	if (pVM->hm.s.fNestedPaging)
1737	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1738	else
1739	{
1740	/*
1741	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1742	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1743	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1744	*/
1745	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1747	}
1748
1749	if (pVM->hm.s.vmx.fVpid)
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1751
1752	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1753	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1754
1755	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1756	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1757	* done dynamically. */
1758	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1759	{
1760	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1761	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1763	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1764	AssertRCReturn(rc, rc);
1765	}
1766
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1768	{
1769	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1770	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1771	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1772	}
1773
1774	if ((val & zap) != val)
1775	{
1776	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1777	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1778	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1779	}
1780
1781	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1782	AssertRCReturn(rc, rc);
1783
1784	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1785	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1786	}
1787
1788	return VINF_SUCCESS;
1789	}
1790
1791
1792	/**
1793	* Sets up miscellaneous (everything other than Pin & Processor-based
1794	* VM-execution) control fields in the VMCS.
1795	*
1796	* @returns VBox status code.
1797	* @param pVM Pointer to the VM.
1798	* @param pVCpu Pointer to the VMCPU.
1799	*/
1800	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1801	{
1802	AssertPtr(pVM);
1803	AssertPtr(pVCpu);
1804
1805	int rc = VERR_GENERAL_FAILURE;
1806
1807	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1808	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1809
1810	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1811
1812	/*
1813	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1814	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1815	* We thus use the exception bitmap to control it rather than use both.
1816	*/
1817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1818	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1819
1820	/** @todo Explore possibility of using IO-bitmaps. */
1821	/* All IO & IOIO instructions cause VM-exits. */
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1823	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1824
1825	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1826	/* Setup MSR autoloading/autostoring. */
1827	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1828	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1830	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1831	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1832	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1833
1834	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1835	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1836	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1837	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1838	#else
1839	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1840	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1841	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1842	#endif
1843
1844	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1845	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1846
1847	/* Setup debug controls */
1848	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1849	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1850	AssertRCReturn(rc, rc);
1851	return rc;
1852	}
1853
1854
1855	/**
1856	* Sets up the initial exception bitmap in the VMCS based on static conditions
1857	* (i.e. conditions that cannot ever change at runtime).
1858	*
1859	* @returns VBox status code.
1860	* @param pVM Pointer to the VM.
1861	* @param pVCpu Pointer to the VMCPU.
1862	*/
1863	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1864	{
1865	AssertPtr(pVM);
1866	AssertPtr(pVCpu);
1867
1868	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1869
1870	uint32_t u32XcptBitmap = 0;
1871
1872	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1873	if (!pVM->hm.s.fNestedPaging)
1874	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1875
1876	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1877	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1878	AssertRCReturn(rc, rc);
1879	return rc;
1880	}
1881
1882
1883	/**
1884	* Sets up the initial guest-state mask. The guest-state mask is consulted
1885	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1886	* for the nested virtualization case (as it would cause a VM-exit).
1887	*
1888	* @param pVCpu Pointer to the VMCPU.
1889	*/
1890	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1891	{
1892	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1893	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1894	return VINF_SUCCESS;
1895	}
1896
1897
1898	/**
1899	* Does per-VM VT-x initialization.
1900	*
1901	* @returns VBox status code.
1902	* @param pVM Pointer to the VM.
1903	*/
1904	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1905	{
1906	LogFlowFunc(("pVM=%p\n", pVM));
1907
1908	int rc = hmR0VmxStructsAlloc(pVM);
1909	if (RT_FAILURE(rc))
1910	{
1911	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1912	return rc;
1913	}
1914
1915	return VINF_SUCCESS;
1916	}
1917
1918
1919	/**
1920	* Does per-VM VT-x termination.
1921	*
1922	* @returns VBox status code.
1923	* @param pVM Pointer to the VM.
1924	*/
1925	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1926	{
1927	LogFlowFunc(("pVM=%p\n", pVM));
1928
1929	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1930	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1931	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1932	#endif
1933	hmR0VmxStructsFree(pVM);
1934	return VINF_SUCCESS;
1935	}
1936
1937
1938	/**
1939	* Sets up the VM for execution under VT-x.
1940	* This function is only called once per-VM during initalization.
1941	*
1942	* @returns VBox status code.
1943	* @param pVM Pointer to the VM.
1944	*/
1945	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1946	{
1947	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1948	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1949
1950	LogFlowFunc(("pVM=%p\n", pVM));
1951
1952	/*
1953	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1954	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1955	*/
1956	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1957	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1958	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1959	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1960	{
1961	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1962	return VERR_INTERNAL_ERROR;
1963	}
1964
1965	/* Initialize these always, see hmR3InitFinalizeR0().*/
1966	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1967	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1968
1969	/* Setup the tagged-TLB flush handlers. */
1970	int rc = hmR0VmxSetupTaggedTlb(pVM);
1971	if (RT_FAILURE(rc))
1972	{
1973	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1974	return rc;
1975	}
1976
1977	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1978	{
1979	PVMCPU pVCpu = &pVM->aCpus[i];
1980	AssertPtr(pVCpu);
1981	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1982
1983	/* Set revision dword at the beginning of the VMCS structure. */
1984	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1985
1986	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1987	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1988	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1989	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1990
1991	/* Load this VMCS as the current VMCS. */
1992	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1993	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1994	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1995
1996	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1997	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1998	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1999
2000	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2001	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2002	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2003
2004	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2017	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2018	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2019	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2020	#endif
2021
2022	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2023	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2024	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2025	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2026
2027	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2028	}
2029
2030	return VINF_SUCCESS;
2031	}
2032
2033
2034	/**
2035	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2036	* the VMCS.
2037	*
2038	* @returns VBox status code.
2039	* @param pVM Pointer to the VM.
2040	* @param pVCpu Pointer to the VMCPU.
2041	*/
2042	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2043	{
2044	RTCCUINTREG uReg = ASMGetCR0();
2045	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2046
2047	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2048	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2049	if (HMVMX_IS_64BIT_HOST_MODE())
2050	{
2051	uint64_t uRegCR3 = hmR0Get64bitCR3();
2052	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2053	}
2054	else
2055	#endif
2056	{
2057	uReg = ASMGetCR3();
2058	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2059	}
2060
2061	uReg = ASMGetCR4();
2062	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2063	AssertRCReturn(rc, rc);
2064	return rc;
2065	}
2066
2067
2068	/**
2069	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2070	* the host-state area in the VMCS.
2071	*
2072	* @returns VBox status code.
2073	* @param pVM Pointer to the VM.
2074	* @param pVCpu Pointer to the VMCPU.
2075	*/
2076	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2077	{
2078	int rc = VERR_INTERNAL_ERROR_5;
2079	RTSEL uSelCS = 0;
2080	RTSEL uSelSS = 0;
2081	RTSEL uSelDS = 0;
2082	RTSEL uSelES = 0;
2083	RTSEL uSelFS = 0;
2084	RTSEL uSelGS = 0;
2085	RTSEL uSelTR = 0;
2086
2087	/*
2088	* Host Selector registers.
2089	*/
2090	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2091	if (HMVMX_IS_64BIT_HOST_MODE())
2092	{
2093	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2094	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2095	}
2096	else
2097	{
2098	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2099	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2100	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2101	}
2102	#else
2103	uSelCS = ASMGetCS();
2104	uSelSS = ASMGetSS();
2105	#endif
2106
2107	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2108	uSelTR = ASMGetTR();
2109
2110	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2111	/** @todo Verify if we have any platform that actually run with DS or ES with
2112	* RPL != 0 in kernel space. */
2113	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2114	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2115	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2116	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2117	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2118	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2119	Assert(uSelCS != 0);
2120	Assert(uSelTR != 0);
2121
2122	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2123	#if 0
2124	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2125	Assert(uSelSS != 0);
2126	#endif
2127
2128	/* Write these host selector fields into the host-state area in the VMCS. */
2129	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2130	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2131	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2132	#if 0
2133	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2134	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2135	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2136	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2137	#endif
2138	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2139	AssertRCReturn(rc, rc);
2140
2141	/*
2142	* Host GDTR and IDTR.
2143	*/
2144	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2145	* be safe to -not- save and restore GDTR and IDTR in the assembly
2146	* code and just do it here and don't care if the limits are zapped on
2147	* VM-exit. */
2148	RTGDTR Gdtr;
2149	RT_ZERO(Gdtr);
2150	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2151	if (HMVMX_IS_64BIT_HOST_MODE())
2152	{
2153	X86XDTR64 Gdtr64;
2154	X86XDTR64 Idtr64;
2155	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2156	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2157	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2158	Gdtr.cbGdt = Gdtr64.cb;
2159	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2160	}
2161	else
2162	#endif
2163	{
2164	RTIDTR Idtr;
2165	ASMGetGDTR(&Gdtr);
2166	ASMGetIDTR(&Idtr);
2167	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2168	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2169	}
2170	AssertRCReturn(rc, rc);
2171
2172	/*
2173	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2174	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2175	*/
2176	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2177	{
2178	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2179	return VERR_VMX_INVALID_HOST_STATE;
2180	}
2181
2182	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2183	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2184	if (HMVMX_IS_64BIT_HOST_MODE())
2185	{
2186	/* We need the 64-bit TR base for hybrid darwin. */
2187	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2189	}
2190	else
2191	#endif
2192	{
2193	uintptr_t uTRBase;
2194	#if HC_ARCH_BITS == 64
2195	uTRBase = X86DESC64_BASE(pDesc);
2196	#else
2197	uTRBase = X86DESC_BASE(pDesc);
2198	#endif
2199	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2200	}
2201	AssertRCReturn(rc, rc);
2202
2203	/*
2204	* Host FS base and GS base.
2205	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2206	* would take care of the bases. In 64-bit, the MSRs come into play.
2207	*/
2208	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2209	if (HMVMX_IS_64BIT_HOST_MODE())
2210	{
2211	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2212	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2213	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2214	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2215	AssertRCReturn(rc, rc);
2216	}
2217	#endif
2218	return rc;
2219	}
2220
2221
2222	/**
2223	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2224	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2225	* the host after every successful VM exit.
2226	*
2227	* @returns VBox status code.
2228	* @param pVM Pointer to the VM.
2229	* @param pVCpu Pointer to the VMCPU.
2230	*/
2231	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2232	{
2233	AssertPtr(pVCpu);
2234	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2235
2236	int rc = VINF_SUCCESS;
2237	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2238	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2239	uint32_t cHostMsrs = 0;
2240	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2241
2242	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2243	{
2244	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2245	pHostMsr->u32Reserved = 0;
2246	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2247	if (CPUMIsGuestInLongMode(pVCpu))
2248	{
2249	/* Must match the EFER value in our 64 bits switcher. */
2250	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2251	}
2252	else
2253	# endif
2254	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2255	pHostMsr++; cHostMsrs++;
2256	}
2257
2258	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2259	if (HMVMX_IS_64BIT_HOST_MODE())
2260	{
2261	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2262	pHostMsr->u32Reserved = 0;
2263	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2264	pHostMsr++; cHostMsrs++;
2265	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2266	pHostMsr->u32Reserved = 0;
2267	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2268	pHostMsr++; cHostMsrs++;
2269	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2270	pHostMsr->u32Reserved = 0;
2271	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2272	pHostMsr++; cHostMsrs++;
2273	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2274	pHostMsr->u32Reserved = 0;
2275	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2276	pHostMsr++; cHostMsrs++;
2277	}
2278	# endif
2279
2280	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2281	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2282	{
2283	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2284	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2285	}
2286
2287	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2288	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2289
2290	/*
2291	* Host Sysenter MSRs.
2292	*/
2293	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2294	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2295	if (HMVMX_IS_64BIT_HOST_MODE())
2296	{
2297	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2298	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2299	}
2300	else
2301	{
2302	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2303	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2304	}
2305	# elif HC_ARCH_BITS == 32
2306	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2308	# else
2309	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2311	# endif
2312	AssertRCReturn(rc, rc);
2313
2314	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2315	* hmR0VmxSetupExitCtls() !! */
2316	return rc;
2317	}
2318
2319
2320	/**
2321	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2322	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2323	* controls".
2324	*
2325	* @returns VBox status code.
2326	* @param pVCpu Pointer to the VMCPU.
2327	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2328	* out-of-sync. Make sure to update the required fields
2329	* before using them.
2330	*
2331	* @remarks No-long-jump zone!!!
2332	*/
2333	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2334	{
2335	int rc = VINF_SUCCESS;
2336	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2337	{
2338	PVM pVM = pVCpu->CTX_SUFF(pVM);
2339	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2340	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2341
2342	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2343	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2344
2345	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2346	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2347	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2348	else
2349	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2350
2351	/*
2352	* The following should not be set (since we're not in SMM mode):
2353	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2354	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2355	*/
2356
2357	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2358	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2359	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2360
2361	if ((val & zap) != val)
2362	{
2363	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2364	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2365	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2366	}
2367
2368	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2369	AssertRCReturn(rc, rc);
2370
2371	/* Update VCPU with the currently set VM-exit controls. */
2372	pVCpu->hm.s.vmx.u32EntryCtls = val;
2373	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2374	}
2375	return rc;
2376	}
2377
2378
2379	/**
2380	* Sets up the VM-exit controls in the VMCS.
2381	*
2382	* @returns VBox status code.
2383	* @param pVM Pointer to the VM.
2384	* @param pVCpu Pointer to the VMCPU.
2385	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2386	* out-of-sync. Make sure to update the required fields
2387	* before using them.
2388	*
2389	* @remarks requires EFER.
2390	*/
2391	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2392	{
2393	int rc = VINF_SUCCESS;
2394	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2395	{
2396	PVM pVM = pVCpu->CTX_SUFF(pVM);
2397	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2398	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2399
2400	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2401	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2402
2403	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2404	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2405	if (HMVMX_IS_64BIT_HOST_MODE())
2406	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2407	else
2408	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2409	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2410	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2411	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2412	else
2413	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2414	#endif
2415
2416	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2417	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2418
2419	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2420	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2421	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2422	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2423	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2424
2425	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2426	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2427
2428	if ((val & zap) != val)
2429	{
2430	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2431	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2432	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2433	}
2434
2435	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2436	AssertRCReturn(rc, rc);
2437
2438	/* Update VCPU with the currently set VM-exit controls. */
2439	pVCpu->hm.s.vmx.u32ExitCtls = val;
2440	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2441	}
2442	return rc;
2443	}
2444
2445
2446	/**
2447	* Loads the guest APIC and related state.
2448	*
2449	* @returns VBox status code.
2450	* @param pVM Pointer to the VM.
2451	* @param pVCpu Pointer to the VMCPU.
2452	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2453	* out-of-sync. Make sure to update the required fields
2454	* before using them.
2455	*/
2456	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2457	{
2458	int rc = VINF_SUCCESS;
2459	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2460	{
2461	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2462	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2463	{
2464	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2465
2466	bool fPendingIntr = false;
2467	uint8_t u8GuestTpr = 0;
2468	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2469	AssertRCReturn(rc, rc);
2470
2471	/*
2472	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2473	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2474	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2475	* the interrupt when we VM-exit for other reasons.
2476	*/
2477	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2478	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2479	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2480	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2481
2482	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2483	AssertRCReturn(rc, rc);
2484
2485	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2486	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2487	{
2488	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2489	pMixedCtx->msrLSTAR = u8GuestTpr;
2490	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2491	{
2492	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2493	if (fPendingIntr)
2494	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2495	else
2496	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497	}
2498	}
2499	}
2500
2501	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2502	}
2503	return rc;
2504	}
2505
2506
2507	/**
2508	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2509	*
2510	* @returns
2511	* @param pVCpu Pointer to the VMCPU.
2512	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2513	* out-of-sync. Make sure to update the required fields
2514	* before using them.
2515	*
2516	* @remarks No-long-jump zone!!!
2517	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2518	*/
2519	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2520	{
2521	/*
2522	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2523	* inhibit interrupts or clear any existing interrupt-inhibition.
2524	*/
2525	uint32_t uIntrState = 0;
2526	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2527	{
2528	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2529	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2530	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2531	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2532	{
2533	/*
2534	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2535	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2536	*/
2537	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2538	}
2539	else if (pMixedCtx->eflags.Bits.u1IF)
2540	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2541	else
2542	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2543	}
2544	return uIntrState;
2545	}
2546
2547
2548	/**
2549	* Loads the guest's interruptibility-state into the guest-state area in the
2550	* VMCS.
2551	*
2552	* @returns VBox status code.
2553	* @param pVCpu Pointer to the VMCPU.
2554	* @param uIntrState The interruptibility-state to set.
2555	*/
2556	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2557	{
2558	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2559	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2560	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2561	AssertRCReturn(rc, rc);
2562	return rc;
2563	}
2564
2565
2566	/**
2567	* Loads the guest's RIP into the guest-state area in the VMCS.
2568	*
2569	* @returns VBox status code.
2570	* @param pVCpu Pointer to the VMCPU.
2571	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2572	* out-of-sync. Make sure to update the required fields
2573	* before using them.
2574	*
2575	* @remarks No-long-jump zone!!!
2576	*/
2577	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2578	{
2579	int rc = VINF_SUCCESS;
2580	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2581	{
2582	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2583	AssertRCReturn(rc, rc);
2584	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2585	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2586	}
2587	return rc;
2588	}
2589
2590
2591	/**
2592	* Loads the guest's RSP into the guest-state area in the VMCS.
2593	*
2594	* @returns VBox status code.
2595	* @param pVCpu Pointer to the VMCPU.
2596	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2597	* out-of-sync. Make sure to update the required fields
2598	* before using them.
2599	*
2600	* @remarks No-long-jump zone!!!
2601	*/
2602	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2603	{
2604	int rc = VINF_SUCCESS;
2605	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2606	{
2607	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2608	AssertRCReturn(rc, rc);
2609	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2610	}
2611	return rc;
2612	}
2613
2614
2615	/**
2616	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2617	*
2618	* @returns VBox status code.
2619	* @param pVCpu Pointer to the VMCPU.
2620	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2621	* out-of-sync. Make sure to update the required fields
2622	* before using them.
2623	*
2624	* @remarks No-long-jump zone!!!
2625	*/
2626	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2627	{
2628	int rc = VINF_SUCCESS;
2629	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2630	{
2631	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2632	Let us assert it as such and use 32-bit VMWRITE. */
2633	Assert(!(pMixedCtx->rflags.u64 >> 32));
2634	X86EFLAGS uEFlags = pMixedCtx->eflags;
2635	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2636	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2637
2638	/*
2639	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2640	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2641	*/
2642	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2643	{
2644	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2645	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2646	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2647	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2648	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2649	}
2650
2651	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2652	AssertRCReturn(rc, rc);
2653
2654	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2656	}
2657	return rc;
2658	}
2659
2660
2661	/**
2662	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2663	*
2664	* @returns VBox status code.
2665	* @param pVCpu Pointer to the VMCPU.
2666	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2667	* out-of-sync. Make sure to update the required fields
2668	* before using them.
2669	*
2670	* @remarks No-long-jump zone!!!
2671	*/
2672	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2673	{
2674	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2675	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2676	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2677	return rc;
2678	}
2679
2680
2681	/**
2682	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2683	* in the VMCS.
2684	*
2685	* @returns VBox status code.
2686	* @param pVM Pointer to the VM.
2687	* @param pVCpu Pointer to the VMCPU.
2688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2689	* out-of-sync. Make sure to update the required fields
2690	* before using them.
2691	*
2692	* @remarks No-long-jump zone!!!
2693	*/
2694	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2695	{
2696	int rc = VINF_SUCCESS;
2697	PVM pVM = pVCpu->CTX_SUFF(pVM);
2698
2699	/*
2700	* Guest CR0.
2701	* Guest FPU.
2702	*/
2703	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2704	{
2705	Assert(!(pCtx->cr0 >> 32));
2706	uint32_t u32GuestCR0 = pCtx->cr0;
2707
2708	/* The guest's view (read access) of its CR0 is unblemished. */
2709	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2710	AssertRCReturn(rc, rc);
2711	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2712
2713	/* Setup VT-x's view of the guest CR0. */
2714	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2715	if (pVM->hm.s.fNestedPaging)
2716	{
2717	if (CPUMIsGuestPagingEnabledEx(pCtx))
2718	{
2719	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2720	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2721	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2722	}
2723	else
2724	{
2725	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2726	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2727	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2728	}
2729
2730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2731	AssertRCReturn(rc, rc);
2732	}
2733	else
2734	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2735
2736	/*
2737	* Guest FPU bits.
2738	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2739	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2740	*/
2741	u32GuestCR0 \|= X86_CR0_NE;
2742	bool fInterceptNM = false;
2743	if (CPUMIsGuestFPUStateActive(pVCpu))
2744	{
2745	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2746	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2747	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2748	}
2749	else
2750	{
2751	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2752	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2753	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2754	}
2755
2756	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2757	bool fInterceptMF = false;
2758	if (!(pCtx->cr0 & X86_CR0_NE))
2759	fInterceptMF = true;
2760
2761	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2762	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2763	{
2764	Assert(PDMVmmDevHeapIsEnabled(pVM));
2765	Assert(pVM->hm.s.vmx.pRealModeTSS);
2766	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2767	fInterceptNM = true;
2768	fInterceptMF = true;
2769	}
2770	else
2771	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2772
2773	if (fInterceptNM)
2774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2775	else
2776	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2777
2778	if (fInterceptMF)
2779	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2780	else
2781	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2782
2783	/* Additional intercepts for debugging, define these yourself explicitly. */
2784	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2786	\| RT_BIT(X86_XCPT_DB)
2787	\| RT_BIT(X86_XCPT_DE)
2788	\| RT_BIT(X86_XCPT_NM)
2789	\| RT_BIT(X86_XCPT_UD)
2790	\| RT_BIT(X86_XCPT_NP)
2791	\| RT_BIT(X86_XCPT_SS)
2792	\| RT_BIT(X86_XCPT_GP)
2793	\| RT_BIT(X86_XCPT_PF)
2794	\| RT_BIT(X86_XCPT_MF);
2795	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2796	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2797	#endif
2798
2799	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2800
2801	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2802	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2803	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2804	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2805	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2806	else
2807	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2808
2809	u32GuestCR0 \|= uSetCR0;
2810	u32GuestCR0 &= uZapCR0;
2811	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2812
2813	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2814	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2815	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2816	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2817
2818	/*
2819	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2820	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2821	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2822	*/
2823	uint64_t u64CR0Mask = 0;
2824	u64CR0Mask = X86_CR0_PE
2825	\| X86_CR0_NE
2826	\| X86_CR0_WP
2827	\| X86_CR0_PG
2828	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2829	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2830	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2831	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2832	u64CR0Mask &= ~X86_CR0_PE;
2833	if (pVM->hm.s.fNestedPaging)
2834	u64CR0Mask &= ~X86_CR0_WP;
2835
2836	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2837	if (fInterceptNM)
2838	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2839	else
2840	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2841
2842	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2843	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2844	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2845	AssertRCReturn(rc, rc);
2846
2847	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2848	}
2849
2850	/*
2851	* Guest CR2.
2852	* It's always loaded in the assembler code. Nothing to do here.
2853	*/
2854
2855	/*
2856	* Guest CR3.
2857	*/
2858	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2859	{
2860	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2861	if (pVM->hm.s.fNestedPaging)
2862	{
2863	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2864
2865	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2866	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2867	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2868	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2869
2870	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2871	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2872	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2873
2874	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2875	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2876	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2877	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2878
2879	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2880	AssertRCReturn(rc, rc);
2881	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2882
2883	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2884	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2885	{
2886	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2887	if (CPUMIsGuestInPAEModeEx(pCtx))
2888	{
2889	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2890	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2891	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2892	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2893	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2894	AssertRCReturn(rc, rc);
2895	}
2896
2897	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2898	have Unrestricted Execution to handle the guest when it's not using paging. */
2899	GCPhysGuestCR3 = pCtx->cr3;
2900	}
2901	else
2902	{
2903	/*
2904	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2905	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2906	* EPT takes care of translating it to host-physical addresses.
2907	*/
2908	RTGCPHYS GCPhys;
2909	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2910	Assert(PDMVmmDevHeapIsEnabled(pVM));
2911
2912	/* We obtain it here every time as the guest could have relocated this PCI region. */
2913	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2914	AssertRCReturn(rc, rc);
2915
2916	GCPhysGuestCR3 = GCPhys;
2917	}
2918	}
2919	else
2920	{
2921	/* Non-nested paging case, just use the hypervisor's CR3. */
2922	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2923	}
2924
2925	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2926	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2927	AssertRCReturn(rc, rc);
2928
2929	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2930	}
2931
2932	/*
2933	* Guest CR4.
2934	*/
2935	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2936	{
2937	Assert(!(pCtx->cr4 >> 32));
2938	uint32_t u32GuestCR4 = pCtx->cr4;
2939
2940	/* The guest's view of its CR4 is unblemished. */
2941	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2942	AssertRCReturn(rc, rc);
2943	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2944
2945	/* Setup VT-x's view of the guest CR4. */
2946	/*
2947	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2948	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2949	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2950	*/
2951	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2952	{
2953	Assert(pVM->hm.s.vmx.pRealModeTSS);
2954	Assert(PDMVmmDevHeapIsEnabled(pVM));
2955	u32GuestCR4 &= ~X86_CR4_VME;
2956	}
2957
2958	if (pVM->hm.s.fNestedPaging)
2959	{
2960	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2961	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2962	{
2963	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2964	u32GuestCR4 \|= X86_CR4_PSE;
2965	/* Our identity mapping is a 32 bits page directory. */
2966	u32GuestCR4 &= ~X86_CR4_PAE;
2967	}
2968	/* else use guest CR4.*/
2969	}
2970	else
2971	{
2972	/*
2973	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2974	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2975	*/
2976	switch (pVCpu->hm.s.enmShadowMode)
2977	{
2978	case PGMMODE_REAL: /* Real-mode. */
2979	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2980	case PGMMODE_32_BIT: /* 32-bit paging. */
2981	{
2982	u32GuestCR4 &= ~X86_CR4_PAE;
2983	break;
2984	}
2985
2986	case PGMMODE_PAE: /* PAE paging. */
2987	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2988	{
2989	u32GuestCR4 \|= X86_CR4_PAE;
2990	break;
2991	}
2992
2993	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2994	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2995	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2996	break;
2997	#endif
2998	default:
2999	AssertFailed();
3000	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3001	}
3002	}
3003
3004	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3005	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3006	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3007	u32GuestCR4 \|= uSetCR4;
3008	u32GuestCR4 &= uZapCR4;
3009
3010	/* Write VT-x's view of the guest CR4 into the VMCS. */
3011	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3012	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3013
3014	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3015	uint64_t u64CR4Mask = 0;
3016	u64CR4Mask = X86_CR4_VME
3017	\| X86_CR4_PAE
3018	\| X86_CR4_PGE
3019	\| X86_CR4_PSE
3020	\| X86_CR4_VMXE;
3021	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3022	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3023	AssertRCReturn(rc, rc);
3024
3025	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3026	}
3027	return rc;
3028	}
3029
3030
3031	/**
3032	* Loads the guest debug registers into the guest-state area in the VMCS.
3033	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3034	*
3035	* @returns VBox status code.
3036	* @param pVCpu Pointer to the VMCPU.
3037	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3038	* out-of-sync. Make sure to update the required fields
3039	* before using them.
3040	*
3041	* @remarks No-long-jump zone!!!
3042	*/
3043	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3044	{
3045	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3046	return VINF_SUCCESS;
3047
3048	#ifdef VBOX_STRICT
3049	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3050	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3051	{
3052	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3053	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3054	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3055	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3056	}
3057	#endif
3058
3059	int rc = VERR_INTERNAL_ERROR_5;
3060	PVM pVM = pVCpu->CTX_SUFF(pVM);
3061	bool fInterceptDB = false;
3062	bool fInterceptMovDRx = false;
3063	if (DBGFIsStepping(pVCpu))
3064	{
3065	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3066	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3067	{
3068	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3069	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3070	AssertRCReturn(rc, rc);
3071	Assert(fInterceptDB == false);
3072	}
3073	else
3074	fInterceptDB = true;
3075	}
3076
3077	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3078	{
3079	if (!CPUMIsHyperDebugStateActive(pVCpu))
3080	{
3081	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3082	AssertRC(rc);
3083	}
3084	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3085	fInterceptMovDRx = true;
3086	}
3087	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3088	{
3089	if (!CPUMIsGuestDebugStateActive(pVCpu))
3090	{
3091	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3092	AssertRC(rc);
3093	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3094	}
3095	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3096	Assert(fInterceptMovDRx == false);
3097	}
3098	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3099	{
3100	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3101	fInterceptMovDRx = true;
3102	}
3103
3104	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3105	if (fInterceptDB)
3106	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3107	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3108	{
3109	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3110	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3111	#endif
3112	}
3113
3114	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3115	if (fInterceptMovDRx)
3116	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3117	else
3118	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3119
3120	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3121	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3122
3123	/* The guest's view of its DR7 is unblemished. */
3124	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3125
3126	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3127	return rc;
3128	}
3129
3130
3131	#ifdef VBOX_STRICT
3132	/**
3133	* Strict function to validate segment registers.
3134	*/
3135	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3136	{
3137	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3138	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3139	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3140	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3141	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3142	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3143	{
3144	/* Protected mode checks */
3145	/* CS */
3146	Assert(pCtx->cs.Attr.n.u1Present);
3147	Assert(!(pCtx->cs.Attr.u & 0xf00));
3148	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3149	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3150	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3151	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3152	\|\| (pCtx->cs.Attr.n.u1Granularity));
3153	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3154	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3155	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3156	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3157	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3158	else
3159	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3160	/* SS */
3161	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3162	{
3163	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3164	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3165	Assert(pCtx->ss.Attr.n.u1Present);
3166	Assert(!(pCtx->ss.Attr.u & 0xf00));
3167	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3168	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3169	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3170	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3171	\|\| (pCtx->ss.Attr.n.u1Granularity));
3172	}
3173	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3174	/* CR0 might not be up-to-date here always, hence disabled. */
3175	#if 0
3176	if (!pCtx->cr0 & X86_CR0_PE)
3177	Assert(!pCtx->ss.Attr.n.u2Dpl);
3178	#endif
3179	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3180	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3181	{
3182	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3183	Assert(pCtx->ds.Attr.n.u1Present);
3184	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3185	Assert(!(pCtx->ds.Attr.u & 0xf00));
3186	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3187	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3188	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3189	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3190	\|\| (pCtx->ds.Attr.n.u1Granularity));
3191	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3192	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3193	}
3194	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3195	{
3196	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3197	Assert(pCtx->es.Attr.n.u1Present);
3198	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3199	Assert(!(pCtx->es.Attr.u & 0xf00));
3200	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3201	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3202	\|\| !(pCtx->es.Attr.n.u1Granularity));
3203	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3204	\|\| (pCtx->es.Attr.n.u1Granularity));
3205	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3206	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3207	}
3208	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3209	{
3210	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3211	Assert(pCtx->fs.Attr.n.u1Present);
3212	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3213	Assert(!(pCtx->fs.Attr.u & 0xf00));
3214	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3215	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3216	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3217	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3218	\|\| (pCtx->fs.Attr.n.u1Granularity));
3219	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3220	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3221	}
3222	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3223	{
3224	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3225	Assert(pCtx->gs.Attr.n.u1Present);
3226	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3227	Assert(!(pCtx->gs.Attr.u & 0xf00));
3228	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3229	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3230	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3231	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3232	\|\| (pCtx->gs.Attr.n.u1Granularity));
3233	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3234	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3235	}
3236	/* 64-bit capable CPUs. */
3237	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3238	Assert(!(pCtx->cs.u64Base >> 32));
3239	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3240	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3241	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3242	# endif
3243	}
3244	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3245	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3246	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3247	{
3248	/* Real and v86 mode checks. */
3249	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3250	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3251	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3252	{
3253	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3254	}
3255	else
3256	{
3257	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3258	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3259	}
3260
3261	/* CS */
3262	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3263	Assert(pCtx->cs.u32Limit == 0xffff);
3264	Assert(u32CSAttr == 0xf3);
3265	/* SS */
3266	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3267	Assert(pCtx->ss.u32Limit == 0xffff);
3268	Assert(u32SSAttr == 0xf3);
3269	/* DS */
3270	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3271	Assert(pCtx->ds.u32Limit == 0xffff);
3272	Assert(u32DSAttr == 0xf3);
3273	/* ES */
3274	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3275	Assert(pCtx->es.u32Limit == 0xffff);
3276	Assert(u32ESAttr == 0xf3);
3277	/* FS */
3278	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3279	Assert(pCtx->fs.u32Limit == 0xffff);
3280	Assert(u32FSAttr == 0xf3);
3281	/* GS */
3282	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3283	Assert(pCtx->gs.u32Limit == 0xffff);
3284	Assert(u32GSAttr == 0xf3);
3285	/* 64-bit capable CPUs. */
3286	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3287	Assert(!(pCtx->cs.u64Base >> 32));
3288	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3289	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3290	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3291	# endif
3292	}
3293	}
3294	#endif /* VBOX_STRICT */
3295
3296
3297	/**
3298	* Writes a guest segment register into the guest-state area in the VMCS.
3299	*
3300	* @returns VBox status code.
3301	* @param pVCpu Pointer to the VMCPU.
3302	* @param idxSel Index of the selector in the VMCS.
3303	* @param idxLimit Index of the segment limit in the VMCS.
3304	* @param idxBase Index of the segment base in the VMCS.
3305	* @param idxAccess Index of the access rights of the segment in the VMCS.
3306	* @param pSelReg Pointer to the segment selector.
3307	* @param pCtx Pointer to the guest-CPU context.
3308	*
3309	* @remarks No-long-jump zone!!!
3310	*/
3311	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3312	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3313	{
3314	int rc;
3315	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3316	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3317	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3318	AssertRCReturn(rc, rc);
3319
3320	uint32_t u32Access = pSelReg->Attr.u;
3321	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3322	{
3323	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3324	u32Access = 0xf3;
3325	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3326	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3327	}
3328	else
3329	{
3330	/*
3331	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3332	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3333	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3334	* loaded in protected-mode have their attribute as 0.
3335	*/
3336	if (!u32Access)
3337	u32Access = HMVMX_SEL_UNUSABLE;
3338	}
3339
3340	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3341	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3342	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3343
3344	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3345	AssertRCReturn(rc, rc);
3346	return rc;
3347	}
3348
3349
3350	/**
3351	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3352	* into the guest-state area in the VMCS.
3353	*
3354	* @returns VBox status code.
3355	* @param pVM Pointer to the VM.
3356	* @param pVCPU Pointer to the VMCPU.
3357	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3358	* out-of-sync. Make sure to update the required fields
3359	* before using them.
3360	*
3361	* @remarks No-long-jump zone!!!
3362	*/
3363	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3364	{
3365	int rc = VERR_INTERNAL_ERROR_5;
3366	PVM pVM = pVCpu->CTX_SUFF(pVM);
3367
3368	/*
3369	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3370	*/
3371	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3372	{
3373	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3374	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3375	{
3376	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3377	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3378	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3379	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3380	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3381	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3382	}
3383
3384	#ifdef VBOX_WITH_REM
3385	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3386	{
3387	Assert(pVM->hm.s.vmx.pRealModeTSS);
3388	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3389	if ( pVCpu->hm.s.vmx.fWasInRealMode
3390	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3391	{
3392	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3393	in real-mode (e.g. OpenBSD 4.0) */
3394	REMFlushTBs(pVM);
3395	Log(("Load: Switch to protected mode detected!\n"));
3396	pVCpu->hm.s.vmx.fWasInRealMode = false;
3397	}
3398	}
3399	#endif
3400	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3401	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3402	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3403	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3404	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3405	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3406	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3407	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3408	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3409	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3410	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3411	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3412	AssertRCReturn(rc, rc);
3413
3414	#ifdef VBOX_STRICT
3415	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3416	#endif
3417	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3418	}
3419
3420	/*
3421	* Guest TR.
3422	*/
3423	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3424	{
3425	/*
3426	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3427	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3428	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3429	*/
3430	uint16_t u16Sel = 0;
3431	uint32_t u32Limit = 0;
3432	uint64_t u64Base = 0;
3433	uint32_t u32AccessRights = 0;
3434
3435	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3436	{
3437	u16Sel = pMixedCtx->tr.Sel;
3438	u32Limit = pMixedCtx->tr.u32Limit;
3439	u64Base = pMixedCtx->tr.u64Base;
3440	u32AccessRights = pMixedCtx->tr.Attr.u;
3441	}
3442	else
3443	{
3444	Assert(pVM->hm.s.vmx.pRealModeTSS);
3445	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3446
3447	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3448	RTGCPHYS GCPhys;
3449	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3450	AssertRCReturn(rc, rc);
3451
3452	X86DESCATTR DescAttr;
3453	DescAttr.u = 0;
3454	DescAttr.n.u1Present = 1;
3455	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3456
3457	u16Sel = 0;
3458	u32Limit = HM_VTX_TSS_SIZE;
3459	u64Base = GCPhys; /* in real-mode phys = virt. */
3460	u32AccessRights = DescAttr.u;
3461	}
3462
3463	/* Validate. */
3464	Assert(!(u16Sel & RT_BIT(2)));
3465	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3466	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3467	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3468	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3469	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3470	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3471	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3472	Assert( (u32Limit & 0xfff) == 0xfff
3473	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3474	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3475	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3476
3477	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3478	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3479	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3480	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3481	AssertRCReturn(rc, rc);
3482
3483	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3484	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3485	}
3486
3487	/*
3488	* Guest GDTR.
3489	*/
3490	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3491	{
3492	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3493	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3494	AssertRCReturn(rc, rc);
3495
3496	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3497	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3498	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3499	}
3500
3501	/*
3502	* Guest LDTR.
3503	*/
3504	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3505	{
3506	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3507	uint32_t u32Access = 0;
3508	if (!pMixedCtx->ldtr.Attr.u)
3509	u32Access = HMVMX_SEL_UNUSABLE;
3510	else
3511	u32Access = pMixedCtx->ldtr.Attr.u;
3512
3513	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3514	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3515	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3516	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3517	AssertRCReturn(rc, rc);
3518
3519	/* Validate. */
3520	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3521	{
3522	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3523	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3524	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3525	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3526	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3527	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3528	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3529	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3530	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3531	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3532	}
3533
3534	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3535	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3536	}
3537
3538	/*
3539	* Guest IDTR.
3540	*/
3541	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3542	{
3543	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3544	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3545	AssertRCReturn(rc, rc);
3546
3547	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3548	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3549	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3550	}
3551
3552	return VINF_SUCCESS;
3553	}
3554
3555
3556	/**
3557	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3558	* areas. These MSRs will automatically be loaded to the host CPU on every
3559	* successful VM entry and stored from the host CPU on every successful VM exit.
3560	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3561	*
3562	* @returns VBox status code.
3563	* @param pVCpu Pointer to the VMCPU.
3564	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3565	* out-of-sync. Make sure to update the required fields
3566	* before using them.
3567	*
3568	* @remarks No-long-jump zone!!!
3569	*/
3570	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3571	{
3572	AssertPtr(pVCpu);
3573	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3574
3575	/*
3576	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3577	*/
3578	int rc = VINF_SUCCESS;
3579	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3580	{
3581	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3582	PVM pVM = pVCpu->CTX_SUFF(pVM);
3583	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3584	uint32_t cGuestMsrs = 0;
3585
3586	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3587	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3588	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3589	if (fSupportsNX \|\| fSupportsLongMode)
3590	{
3591	/** @todo support save IA32_EFER, i.e.
3592	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3593	* guest EFER need not be part of the VM-entry MSR-load area. */
3594	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3595	pGuestMsr->u32Reserved = 0;
3596	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3597	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3598	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3599	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3600	pGuestMsr++; cGuestMsrs++;
3601	if (fSupportsLongMode)
3602	{
3603	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3604	pGuestMsr->u32Reserved = 0;
3605	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3606	pGuestMsr++; cGuestMsrs++;
3607	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3608	pGuestMsr->u32Reserved = 0;
3609	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3610	pGuestMsr++; cGuestMsrs++;
3611	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3612	pGuestMsr->u32Reserved = 0;
3613	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3614	pGuestMsr++; cGuestMsrs++;
3615	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3618	pGuestMsr++; cGuestMsrs++;
3619	}
3620	}
3621
3622	/*
3623	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3624	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3625	*/
3626	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3627	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3628	{
3629	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3630	pGuestMsr->u32Reserved = 0;
3631	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3632	AssertRCReturn(rc, rc);
3633	pGuestMsr++; cGuestMsrs++;
3634	}
3635
3636	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3637	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3638	{
3639	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3640	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3641	}
3642
3643	/* Update the VCPU's copy of the guest MSR count. */
3644	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3645	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3646	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3647	AssertRCReturn(rc, rc);
3648	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3649
3650	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3651	}
3652
3653	/*
3654	* Guest Sysenter MSRs.
3655	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3656	* VM-exits on WRMSRs for these MSRs.
3657	*/
3658	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3659	{
3660	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3661	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3662	}
3663	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3664	{
3665	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3666	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3667	}
3668	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3669	{
3670	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3671	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3672	}
3673	AssertRCReturn(rc, rc);
3674
3675	return rc;
3676	}
3677
3678
3679	/**
3680	* Loads the guest activity state into the guest-state area in the VMCS.
3681	*
3682	* @returns VBox status code.
3683	* @param pVCpu Pointer to the VMCPU.
3684	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3685	* out-of-sync. Make sure to update the required fields
3686	* before using them.
3687	*
3688	* @remarks No-long-jump zone!!!
3689	*/
3690	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3691	{
3692	/** @todo See if we can make use of other states, e.g.
3693	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3694	int rc = VINF_SUCCESS;
3695	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3696	{
3697	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3698	AssertRCReturn(rc, rc);
3699	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3700	}
3701	return rc;
3702	}
3703
3704
3705	/**
3706	* Sets up the appropriate function to run guest code.
3707	*
3708	* @returns VBox status code.
3709	* @param pVCpu Pointer to the VMCPU.
3710	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3711	* out-of-sync. Make sure to update the required fields
3712	* before using them.
3713	*
3714	* @remarks No-long-jump zone!!!
3715	*/
3716	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3717	{
3718	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3719	{
3720	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3721	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3722	#endif
3723	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3724	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3725	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3726	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3727	#else
3728	/* 64-bit host or hybrid host. */
3729	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3730	#endif
3731	}
3732	else
3733	{
3734	/* Guest is not in long mode, use the 32-bit handler. */
3735	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3736	}
3737	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3738	return VINF_SUCCESS;
3739	}
3740
3741
3742	/**
3743	* Wrapper for running the guest code in VT-x.
3744	*
3745	* @returns VBox strict status code.
3746	* @param pVM Pointer to the VM.
3747	* @param pVCpu Pointer to the VMCPU.
3748	* @param pCtx Pointer to the guest-CPU context.
3749	*
3750	* @remarks No-long-jump zone!!!
3751	*/
3752	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3753	{
3754	/*
3755	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3756	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3757	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3758	*/
3759	#ifdef VBOX_WITH_KERNEL_USING_XMM
3760	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3761	#else
3762	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3763	#endif
3764	}
3765
3766
3767	/**
3768	* Report world-switch error and dump some useful debug info.
3769	*
3770	* @param pVM Pointer to the VM.
3771	* @param pVCpu Pointer to the VMCPU.
3772	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3773	* @param pCtx Pointer to the guest-CPU context.
3774	* @param pVmxTransient Pointer to the VMX transient structure (only
3775	* exitReason updated).
3776	*/
3777	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3778	{
3779	Assert(pVM);
3780	Assert(pVCpu);
3781	Assert(pCtx);
3782	Assert(pVmxTransient);
3783	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3784
3785	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3786	switch (rcVMRun)
3787	{
3788	case VERR_VMX_INVALID_VMXON_PTR:
3789	AssertFailed();
3790	break;
3791	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3792	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3793	{
3794	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3795	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3796	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3797	AssertRC(rc);
3798
3799	#ifdef VBOX_STRICT
3800	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3801	pVmxTransient->uExitReason));
3802	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3803	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3804	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3805	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3806	else
3807	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3808
3809	/* VMX control bits. */
3810	uint32_t u32Val;
3811	uint64_t u64Val;
3812	HMVMXHCUINTREG uHCReg;
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3845	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3846	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3847	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3848	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3849	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3850	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3852	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3853	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3854	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3855
3856	/* Guest bits. */
3857	RTGCUINTREG uGCReg;
3858	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3859	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3860	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3861	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3862	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3863	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3864	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3865	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3866
3867	/* Host bits. */
3868	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3869	Log(("Host CR0 %#RHr\n", uHCReg));
3870	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3871	Log(("Host CR3 %#RHr\n", uHCReg));
3872	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3873	Log(("Host CR4 %#RHr\n", uHCReg));
3874
3875	RTGDTR HostGdtr;
3876	PCX86DESCHC pDesc;
3877	ASMGetGDTR(&HostGdtr);
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3879	Log(("Host CS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3887	Log(("Host DS %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3892	}
3893
3894	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3895	Log(("Host ES %#08x\n", u32Val));
3896	if (u32Val < HostGdtr.cbGdt)
3897	{
3898	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3899	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3900	}
3901
3902	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3903	Log(("Host FS %#08x\n", u32Val));
3904	if (u32Val < HostGdtr.cbGdt)
3905	{
3906	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3907	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3908	}
3909
3910	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3911	Log(("Host GS %#08x\n", u32Val));
3912	if (u32Val < HostGdtr.cbGdt)
3913	{
3914	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3915	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3916	}
3917
3918	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3919	Log(("Host SS %#08x\n", u32Val));
3920	if (u32Val < HostGdtr.cbGdt)
3921	{
3922	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3923	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3924	}
3925
3926	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3927	Log(("Host TR %#08x\n", u32Val));
3928	if (u32Val < HostGdtr.cbGdt)
3929	{
3930	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3931	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3932	}
3933
3934	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3935	Log(("Host TR Base %#RHv\n", uHCReg));
3936	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3937	Log(("Host GDTR Base %#RHv\n", uHCReg));
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3939	Log(("Host IDTR Base %#RHv\n", uHCReg));
3940	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3941	Log(("Host SYSENTER CS %#08x\n", u32Val));
3942	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3943	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3944	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3945	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3947	Log(("Host RSP %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3949	Log(("Host RIP %#RHv\n", uHCReg));
3950	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3951	if (HMVMX_IS_64BIT_HOST_MODE())
3952	{
3953	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3954	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3955	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3956	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3957	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3958	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3959	}
3960	# endif
3961	#endif /* VBOX_STRICT */
3962	break;
3963	}
3964
3965	default:
3966	/* Impossible */
3967	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3968	break;
3969	}
3970	NOREF(pVM);
3971	}
3972
3973
3974	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3975	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3976	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3977	#endif
3978	#ifdef VBOX_STRICT
3979	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3980	{
3981	switch (idxField)
3982	{
3983	case VMX_VMCS_GUEST_RIP:
3984	case VMX_VMCS_GUEST_RSP:
3985	case VMX_VMCS_GUEST_DR7:
3986	case VMX_VMCS_GUEST_SYSENTER_EIP:
3987	case VMX_VMCS_GUEST_SYSENTER_ESP:
3988	case VMX_VMCS_GUEST_GDTR_BASE:
3989	case VMX_VMCS_GUEST_IDTR_BASE:
3990	case VMX_VMCS_GUEST_CS_BASE:
3991	case VMX_VMCS_GUEST_DS_BASE:
3992	case VMX_VMCS_GUEST_ES_BASE:
3993	case VMX_VMCS_GUEST_FS_BASE:
3994	case VMX_VMCS_GUEST_GS_BASE:
3995	case VMX_VMCS_GUEST_SS_BASE:
3996	case VMX_VMCS_GUEST_LDTR_BASE:
3997	case VMX_VMCS_GUEST_TR_BASE:
3998	case VMX_VMCS_GUEST_CR3:
3999	return true;
4000	}
4001	return false;
4002	}
4003
4004	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4005	{
4006	switch (idxField)
4007	{
4008	/* Read-only fields. */
4009	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4010	return true;
4011	}
4012	/* All readable fields should also be part of the VMCS write cache. */
4013	return hmR0VmxIsValidWriteField(idxField);
4014	}
4015	#endif /* VBOX_STRICT */
4016
4017	/**
4018	* Executes the specified handler in 64-bit mode.
4019	*
4020	* @returns VBox status code.
4021	* @param pVM Pointer to the VM.
4022	* @param pVCpu Pointer to the VMCPU.
4023	* @param pCtx Pointer to the guest CPU context.
4024	* @param enmOp The operation to perform.
4025	* @param cbParam Number of parameters.
4026	* @param paParam Array of 32-bit parameters.
4027	*/
4028	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4029	uint32_t *paParam)
4030	{
4031	int rc, rc2;
4032	PHMGLOBLCPUINFO pCpu;
4033	RTHCPHYS HCPhysCpuPage;
4034	RTCCUINTREG uOldEFlags;
4035
4036	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4037	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4038	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4039	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4040
4041	#ifdef VBOX_STRICT
4042	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4043	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4044
4045	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4046	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4047	#endif
4048
4049	/* Disable interrupts. */
4050	uOldEFlags = ASMIntDisableFlags();
4051
4052	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4053	RTCPUID idHostCpu = RTMpCpuId();
4054	CPUMR0SetLApic(pVM, idHostCpu);
4055	#endif
4056
4057	pCpu = HMR0GetCurrentCpu();
4058	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4059
4060	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4061	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4062
4063	/* Leave VMX Root Mode. */
4064	VMXDisable();
4065
4066	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4067
4068	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4069	CPUMSetHyperEIP(pVCpu, enmOp);
4070	for (int i = (int)cbParam - 1; i >= 0; i--)
4071	CPUMPushHyper(pVCpu, paParam[i]);
4072
4073	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4074
4075	/* Call the switcher. */
4076	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4077	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4078
4079	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4080	/* Make sure the VMX instructions don't cause #UD faults. */
4081	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4082
4083	/* Re-enter VMX Root Mode */
4084	rc2 = VMXEnable(HCPhysCpuPage);
4085	if (RT_FAILURE(rc2))
4086	{
4087	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4088	ASMSetFlags(uOldEFlags);
4089	return rc2;
4090	}
4091
4092	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4093	AssertRC(rc2);
4094	Assert(!(ASMGetFlags() & X86_EFL_IF));
4095	ASMSetFlags(uOldEFlags);
4096	return rc;
4097	}
4098
4099
4100	/**
4101	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4102	* supporting 64-bit guests.
4103	*
4104	* @returns VBox status code.
4105	* @param fResume Whether to VMLAUNCH or VMRESUME.
4106	* @param pCtx Pointer to the guest-CPU context.
4107	* @param pCache Pointer to the VMCS cache.
4108	* @param pVM Pointer to the VM.
4109	* @param pVCpu Pointer to the VMCPU.
4110	*/
4111	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4112	{
4113	uint32_t aParam[6];
4114	PHMGLOBLCPUINFO pCpu = NULL;
4115	RTHCPHYS HCPhysCpuPage = 0;
4116	int rc = VERR_INTERNAL_ERROR_5;
4117	AssertReturn(pVM->hm.s.pfnVMXGCStartVM64, VERR_HM_IPE_5);
4118
4119	pCpu = HMR0GetCurrentCpu();
4120	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4121
4122	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4123	pCache->uPos = 1;
4124	pCache->interPD = PGMGetInterPaeCR3(pVM);
4125	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4126	#endif
4127
4128	#ifdef VBOX_STRICT
4129	pCache->TestIn.HCPhysCpuPage = 0;
4130	pCache->TestIn.HCPhysVmcs = 0;
4131	pCache->TestIn.pCache = 0;
4132	pCache->TestOut.HCPhysVmcs = 0;
4133	pCache->TestOut.pCache = 0;
4134	pCache->TestOut.pCtx = 0;
4135	pCache->TestOut.eflags = 0;
4136	#endif
4137
4138	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4139	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4140	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4141	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4142	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4143	aParam[5] = 0;
4144
4145	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4146	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4147	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4148	#endif
4149	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4150
4151	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4152	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4153	Assert(pCtx->dr[4] == 10);
4154	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4155	#endif
4156
4157	#ifdef VBOX_STRICT
4158	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4159	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4160	pVCpu->hm.s.vmx.HCPhysVmcs));
4161	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4162	pCache->TestOut.HCPhysVmcs));
4163	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4164	pCache->TestOut.pCache));
4165	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4166	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4167	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4168	pCache->TestOut.pCtx));
4169	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4170	#endif
4171	return rc;
4172	}
4173
4174
4175	/**
4176	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4177	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4178	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4179	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4180	*
4181	* @returns VBox status code.
4182	* @param pVM Pointer to the VM.
4183	* @param pVCpu Pointer to the VMCPU.
4184	*/
4185	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4186	{
4187	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4188	{ \
4189	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4190	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4191	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4192	++cReadFields; \
4193	}
4194
4195	AssertPtr(pVM);
4196	AssertPtr(pVCpu);
4197	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4198	uint32_t cReadFields = 0;
4199
4200	/* Guest-natural selector base fields */
4201	#if 0
4202	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4204	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4205	#endif
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4208	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4209	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4210	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4214	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4218	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4219	#if 0
4220	/* Unused natural width guest-state fields. */
4221	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4222	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4223	#endif
4224	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4225	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4226
4227	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4228	#if 0
4229	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4236	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4237	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4238	#endif
4239
4240	/* Natural width guest-state fields. */
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4242	#if 0
4243	/* Currently unused field. */
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4245	#endif
4246
4247	if (pVM->hm.s.fNestedPaging)
4248	{
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4250	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4251	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4252	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4253	}
4254	else
4255	{
4256	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4257	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4258	}
4259
4260	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4261	return VINF_SUCCESS;
4262	}
4263
4264
4265	/**
4266	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4267	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4268	* darwin, running 64-bit guests).
4269	*
4270	* @returns VBox status code.
4271	* @param pVCpu Pointer to the VMCPU.
4272	* @param idxField The VMCS field encoding.
4273	* @param u64Val 16, 32 or 64 bits value.
4274	*/
4275	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4276	{
4277	int rc;
4278	switch (idxField)
4279	{
4280	/*
4281	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4282	*/
4283	/* 64-bit Control fields. */
4284	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4285	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4286	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4287	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4288	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4289	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4290	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4291	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4292	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4293	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4294	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4295	case VMX_VMCS64_CTRL_EPTP_FULL:
4296	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4297	/* 64-bit Guest-state fields. */
4298	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4299	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4300	case VMX_VMCS64_GUEST_PAT_FULL:
4301	case VMX_VMCS64_GUEST_EFER_FULL:
4302	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4303	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4304	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4305	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4306	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4307	/* 64-bit Host-state fields. */
4308	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4309	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4310	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4311	{
4312	rc = VMXWriteVmcs32(idxField, u64Val);
4313	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4314	break;
4315	}
4316
4317	/*
4318	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4319	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4320	*/
4321	/* Natural-width Guest-state fields. */
4322	case VMX_VMCS_GUEST_CR3:
4323	case VMX_VMCS_GUEST_ES_BASE:
4324	case VMX_VMCS_GUEST_CS_BASE:
4325	case VMX_VMCS_GUEST_SS_BASE:
4326	case VMX_VMCS_GUEST_DS_BASE:
4327	case VMX_VMCS_GUEST_FS_BASE:
4328	case VMX_VMCS_GUEST_GS_BASE:
4329	case VMX_VMCS_GUEST_LDTR_BASE:
4330	case VMX_VMCS_GUEST_TR_BASE:
4331	case VMX_VMCS_GUEST_GDTR_BASE:
4332	case VMX_VMCS_GUEST_IDTR_BASE:
4333	case VMX_VMCS_GUEST_DR7:
4334	case VMX_VMCS_GUEST_RSP:
4335	case VMX_VMCS_GUEST_RIP:
4336	case VMX_VMCS_GUEST_SYSENTER_ESP:
4337	case VMX_VMCS_GUEST_SYSENTER_EIP:
4338	{
4339	if (!(u64Val >> 32))
4340	{
4341	/* If this field is 64-bit, VT-x will zero out the top bits. */
4342	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4343	}
4344	else
4345	{
4346	/* Assert that only the 32->64 switcher case should ever come here. */
4347	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4348	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4349	}
4350	break;
4351	}
4352
4353	default:
4354	{
4355	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4356	rc = VERR_INVALID_PARAMETER;
4357	break;
4358	}
4359	}
4360	AssertRCReturn(rc, rc);
4361	return rc;
4362	}
4363
4364
4365	/**
4366	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4367	* hosts (except darwin) for 64-bit guests.
4368	*
4369	* @param pVCpu Pointer to the VMCPU.
4370	* @param idxField The VMCS field encoding.
4371	* @param u64Val 16, 32 or 64 bits value.
4372	*/
4373	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4374	{
4375	AssertPtr(pVCpu);
4376	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4377
4378	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4379	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4380
4381	/* Make sure there are no duplicates. */
4382	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4383	{
4384	if (pCache->Write.aField[i] == idxField)
4385	{
4386	pCache->Write.aFieldVal[i] = u64Val;
4387	return VINF_SUCCESS;
4388	}
4389	}
4390
4391	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4392	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4393	pCache->Write.cValidEntries++;
4394	return VINF_SUCCESS;
4395	}
4396
4397	/* Enable later when the assembly code uses these as callbacks. */
4398	#if 0
4399	/*
4400	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4401	*
4402	* @param pVCpu Pointer to the VMCPU.
4403	* @param pCache Pointer to the VMCS cache.
4404	*
4405	* @remarks No-long-jump zone!!!
4406	*/
4407	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4408	{
4409	AssertPtr(pCache);
4410	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4411	{
4412	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4413	AssertRC(rc);
4414	}
4415	pCache->Write.cValidEntries = 0;
4416	}
4417
4418
4419	/**
4420	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4421	*
4422	* @param pVCpu Pointer to the VMCPU.
4423	* @param pCache Pointer to the VMCS cache.
4424	*
4425	* @remarks No-long-jump zone!!!
4426	*/
4427	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4428	{
4429	AssertPtr(pCache);
4430	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4431	{
4432	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4433	AssertRC(rc);
4434	}
4435	}
4436	#endif
4437	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4438
4439
4440	/**
4441	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4442	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4443	* timer.
4444	*
4445	* @returns VBox status code.
4446	* @param pVCpu Pointer to the VMCPU.
4447	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4448	* out-of-sync. Make sure to update the required fields
4449	* before using them.
4450	* @remarks No-long-jump zone!!!
4451	*/
4452	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4453	{
4454	int rc = VERR_INTERNAL_ERROR_5;
4455	bool fOffsettedTsc = false;
4456	PVM pVM = pVCpu->CTX_SUFF(pVM);
4457	if (pVM->hm.s.vmx.fUsePreemptTimer)
4458	{
4459	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4460
4461	/* Make sure the returned values have sane upper and lower boundaries. */
4462	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4463	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4464	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4465	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4466
4467	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4468	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4469	}
4470	else
4471	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4472
4473	if (fOffsettedTsc)
4474	{
4475	uint64_t u64CurTSC = ASMReadTSC();
4476	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4477	{
4478	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4479	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4480
4481	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4482	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4483	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4484	}
4485	else
4486	{
4487	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4488	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4489	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4490	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4491	}
4492	}
4493	else
4494	{
4495	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4496	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4497	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4498	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4499	}
4500	}
4501
4502
4503	/**
4504	* Determines if an exception is a contributory exception. Contributory
4505	* exceptions are ones which can cause double-faults. Page-fault is
4506	* intentionally not included here as it's a conditional contributory exception.
4507	*
4508	* @returns true if the exception is contributory, false otherwise.
4509	* @param uVector The exception vector.
4510	*/
4511	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4512	{
4513	switch (uVector)
4514	{
4515	case X86_XCPT_GP:
4516	case X86_XCPT_SS:
4517	case X86_XCPT_NP:
4518	case X86_XCPT_TS:
4519	case X86_XCPT_DE:
4520	return true;
4521	default:
4522	break;
4523	}
4524	return false;
4525	}
4526
4527
4528	/**
4529	* Sets an event as a pending event to be injected into the guest.
4530	*
4531	* @param pVCpu Pointer to the VMCPU.
4532	* @param u32IntrInfo The VM-entry interruption-information field.
4533	* @param cbInstr The VM-entry instruction length in bytes (for software
4534	* interrupts, exceptions and privileged software
4535	* exceptions).
4536	* @param u32ErrCode The VM-entry exception error code.
4537	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4538	* page-fault.
4539	*/
4540	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4541	RTGCUINTPTR GCPtrFaultAddress)
4542	{
4543	Assert(!pVCpu->hm.s.Event.fPending);
4544	pVCpu->hm.s.Event.fPending = true;
4545	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4546	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4547	pVCpu->hm.s.Event.cbInstr = cbInstr;
4548	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4549	}
4550
4551
4552	/**
4553	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4554	*
4555	* @param pVCpu Pointer to the VMCPU.
4556	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4557	* out-of-sync. Make sure to update the required fields
4558	* before using them.
4559	*/
4560	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4561	{
4562	/* Inject the double-fault. */
4563	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4564	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4565	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4566	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4567	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4568	}
4569
4570
4571	/**
4572	* Handle a condition that occurred while delivering an event through the guest
4573	* IDT.
4574	*
4575	* @returns VBox status code (informational error codes included).
4576	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4577	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4578	* continue execution of the guest which will delivery the #DF.
4579	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4580	*
4581	* @param pVCpu Pointer to the VMCPU.
4582	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4583	* out-of-sync. Make sure to update the required fields
4584	* before using them.
4585	* @param pVmxTransient Pointer to the VMX transient structure.
4586	*
4587	* @remarks No-long-jump zone!!!
4588	*/
4589	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4590	{
4591	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4592	AssertRC(rc);
4593	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4594	{
4595	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4596	AssertRCReturn(rc, rc);
4597
4598	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4599	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4600	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4601
4602	typedef enum
4603	{
4604	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4605	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4606	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4607	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4608	} VMXREFLECTXCPT;
4609
4610	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4611	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4612	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4613	{
4614	enmReflect = VMXREFLECTXCPT_XCPT;
4615	#ifdef VBOX_STRICT
4616	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4617	&& uExitVector == X86_XCPT_PF)
4618	{
4619	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4620	}
4621	#endif
4622	if ( uExitVector == X86_XCPT_PF
4623	&& uIdtVector == X86_XCPT_PF)
4624	{
4625	pVmxTransient->fVectoringPF = true;
4626	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4627	}
4628	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4629	&& hmR0VmxIsContributoryXcpt(uExitVector)
4630	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4631	\|\| uIdtVector == X86_XCPT_PF))
4632	{
4633	enmReflect = VMXREFLECTXCPT_DF;
4634	}
4635	else if (uIdtVector == X86_XCPT_DF)
4636	enmReflect = VMXREFLECTXCPT_TF;
4637	}
4638	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4639	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4640	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4641	{
4642	/*
4643	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4644	* (whatever they are) as they reoccur when restarting the instruction.
4645	*/
4646	enmReflect = VMXREFLECTXCPT_XCPT;
4647	}
4648
4649	switch (enmReflect)
4650	{
4651	case VMXREFLECTXCPT_XCPT:
4652	{
4653	uint32_t u32ErrCode = 0;
4654	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4655	{
4656	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4657	AssertRCReturn(rc, rc);
4658	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4659	}
4660
4661	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4662	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4663	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4664	rc = VINF_SUCCESS;
4665	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4666	pVCpu->hm.s.Event.u32ErrCode));
4667	break;
4668	}
4669
4670	case VMXREFLECTXCPT_DF:
4671	{
4672	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4673	rc = VINF_VMX_DOUBLE_FAULT;
4674	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4675	uIdtVector, uExitVector));
4676	break;
4677	}
4678
4679	case VMXREFLECTXCPT_TF:
4680	{
4681	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4682	rc = VINF_EM_RESET;
4683	break;
4684	}
4685
4686	default:
4687	Assert(rc == VINF_SUCCESS);
4688	break;
4689	}
4690	}
4691	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4692	return rc;
4693	}
4694
4695
4696	/**
4697	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4698	*
4699	* @returns VBox status code.
4700	* @param pVCpu Pointer to the VMCPU.
4701	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4702	* out-of-sync. Make sure to update the required fields
4703	* before using them.
4704	*
4705	* @remarks No-long-jump zone!!!
4706	*/
4707	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4708	{
4709	int rc = VINF_SUCCESS;
4710	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4711	{
4712	uint32_t uVal = 0;
4713	uint32_t uShadow = 0;
4714	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4715	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4716	AssertRCReturn(rc, rc);
4717	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4718	CPUMSetGuestCR0(pVCpu, uVal);
4719	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4720	}
4721	return rc;
4722	}
4723
4724
4725	/**
4726	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4727	*
4728	* @returns VBox status code.
4729	* @param pVCpu Pointer to the VMCPU.
4730	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4731	* out-of-sync. Make sure to update the required fields
4732	* before using them.
4733	*
4734	* @remarks No-long-jump zone!!!
4735	*/
4736	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4737	{
4738	int rc = VINF_SUCCESS;
4739	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4740	{
4741	uint32_t uVal = 0;
4742	uint32_t uShadow = 0;
4743	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4744	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4745	AssertRCReturn(rc, rc);
4746	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4747	CPUMSetGuestCR4(pVCpu, uVal);
4748	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4749	}
4750	return rc;
4751	}
4752
4753
4754	/**
4755	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4756	*
4757	* @returns VBox status code.
4758	* @param pVCpu Pointer to the VMCPU.
4759	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4760	* out-of-sync. Make sure to update the required fields
4761	* before using them.
4762	*
4763	* @remarks No-long-jump zone!!!
4764	*/
4765	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4766	{
4767	int rc = VINF_SUCCESS;
4768	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4769	{
4770	RTGCUINTREG uVal = 0;
4771	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4772	AssertRCReturn(rc, rc);
4773	pMixedCtx->rip = uVal;
4774	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4775	}
4776	return rc;
4777	}
4778
4779
4780	/**
4781	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4782	*
4783	* @returns VBox status code.
4784	* @param pVCpu Pointer to the VMCPU.
4785	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4786	* out-of-sync. Make sure to update the required fields
4787	* before using them.
4788	*
4789	* @remarks No-long-jump zone!!!
4790	*/
4791	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4792	{
4793	int rc = VINF_SUCCESS;
4794	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4795	{
4796	RTGCUINTREG uVal = 0;
4797	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4798	AssertRCReturn(rc, rc);
4799	pMixedCtx->rsp = uVal;
4800	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4801	}
4802	return rc;
4803	}
4804
4805
4806	/**
4807	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4808	*
4809	* @returns VBox status code.
4810	* @param pVCpu Pointer to the VMCPU.
4811	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4812	* out-of-sync. Make sure to update the required fields
4813	* before using them.
4814	*
4815	* @remarks No-long-jump zone!!!
4816	*/
4817	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4818	{
4819	int rc = VINF_SUCCESS;
4820	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4821	{
4822	uint32_t uVal = 0;
4823	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4824	AssertRCReturn(rc, rc);
4825	pMixedCtx->eflags.u32 = uVal;
4826
4827	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4828	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4829	{
4830	PVM pVM = pVCpu->CTX_SUFF(pVM);
4831	Assert(pVM->hm.s.vmx.pRealModeTSS);
4832	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4833	pMixedCtx->eflags.Bits.u1VM = 0;
4834	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4835	}
4836
4837	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4838	}
4839	return rc;
4840	}
4841
4842
4843	/**
4844	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4845	* guest-CPU context.
4846	*/
4847	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4848	{
4849	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4850	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4851	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4852	return rc;
4853	}
4854
4855
4856	/**
4857	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4858	* from the guest-state area in the VMCS.
4859	*
4860	* @param pVCpu Pointer to the VMCPU.
4861	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4862	* out-of-sync. Make sure to update the required fields
4863	* before using them.
4864	*
4865	* @remarks No-long-jump zone!!!
4866	*/
4867	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4868	{
4869	uint32_t uIntrState = 0;
4870	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4871	AssertRC(rc);
4872
4873	if (!uIntrState)
4874	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4875	else
4876	{
4877	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4878	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4879	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4880	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4881	AssertRC(rc);
4882	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4883	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4884	}
4885	}
4886
4887
4888	/**
4889	* Saves the guest's activity state.
4890	*
4891	* @returns VBox status code.
4892	* @param pVCpu Pointer to the VMCPU.
4893	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4894	* out-of-sync. Make sure to update the required fields
4895	* before using them.
4896	*
4897	* @remarks No-long-jump zone!!!
4898	*/
4899	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4900	{
4901	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4902	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4903	return VINF_SUCCESS;
4904	}
4905
4906
4907	/**
4908	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4909	* the current VMCS into the guest-CPU context.
4910	*
4911	* @returns VBox status code.
4912	* @param pVCpu Pointer to the VMCPU.
4913	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4914	* out-of-sync. Make sure to update the required fields
4915	* before using them.
4916	*
4917	* @remarks No-long-jump zone!!!
4918	*/
4919	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4920	{
4921	int rc = VINF_SUCCESS;
4922	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4923	{
4924	uint32_t u32Val = 0;
4925	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4926	pMixedCtx->SysEnter.cs = u32Val;
4927	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4928	}
4929
4930	RTGCUINTREG uGCVal = 0;
4931	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4932	{
4933	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4934	pMixedCtx->SysEnter.eip = uGCVal;
4935	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4936	}
4937	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4938	{
4939	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4940	pMixedCtx->SysEnter.esp = uGCVal;
4941	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4942	}
4943	return rc;
4944	}
4945
4946
4947	/**
4948	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4949	* context.
4950	*
4951	* @returns VBox status code.
4952	* @param pVCpu Pointer to the VMCPU.
4953	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4954	* out-of-sync. Make sure to update the required fields
4955	* before using them.
4956	*
4957	* @remarks No-long-jump zone!!!
4958	*/
4959	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4960	{
4961	int rc = VINF_SUCCESS;
4962	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4963	{
4964	RTGCUINTREG uVal = 0;
4965	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4966	pMixedCtx->fs.u64Base = uVal;
4967	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4968	}
4969	return rc;
4970	}
4971
4972
4973	/**
4974	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4975	* context.
4976	*
4977	* @returns VBox status code.
4978	* @param pVCpu Pointer to the VMCPU.
4979	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4980	* out-of-sync. Make sure to update the required fields
4981	* before using them.
4982	*
4983	* @remarks No-long-jump zone!!!
4984	*/
4985	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4986	{
4987	int rc = VINF_SUCCESS;
4988	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
4989	{
4990	RTGCUINTREG uVal = 0;
4991	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4992	pMixedCtx->gs.u64Base = uVal;
4993	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
4994	}
4995	return rc;
4996	}
4997
4998
4999	/**
5000	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5001	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5002	*
5003	* @returns VBox status code.
5004	* @param pVCpu Pointer to the VMCPU.
5005	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5006	* out-of-sync. Make sure to update the required fields
5007	* before using them.
5008	*
5009	* @remarks No-long-jump zone!!!
5010	*/
5011	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5012	{
5013	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5014	return VINF_SUCCESS;
5015
5016	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5017	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5018	{
5019	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5020	pMsr += i;
5021	switch (pMsr->u32IndexMSR)
5022	{
5023	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5024	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5025	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5026	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5027	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5028	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5029	default:
5030	{
5031	AssertFailed();
5032	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5033	}
5034	}
5035	}
5036	#endif
5037
5038	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5039	return VINF_SUCCESS;
5040	}
5041
5042
5043	/**
5044	* Saves the guest control registers from the current VMCS into the guest-CPU
5045	* context.
5046	*
5047	* @returns VBox status code.
5048	* @param pVCpu Pointer to the VMCPU.
5049	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5050	* out-of-sync. Make sure to update the required fields
5051	* before using them.
5052	*
5053	* @remarks No-long-jump zone!!!
5054	*/
5055	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5056	{
5057	/* Guest CR0. Guest FPU. */
5058	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5059
5060	/* Guest CR4. */
5061	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5062	AssertRCReturn(rc, rc);
5063
5064	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5065	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5066	{
5067	PVM pVM = pVCpu->CTX_SUFF(pVM);
5068	if ( pVM->hm.s.fNestedPaging
5069	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5070	{
5071	RTGCUINTREG uVal = 0;
5072	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5073	if (pMixedCtx->cr3 != uVal)
5074	{
5075	CPUMSetGuestCR3(pVCpu, uVal);
5076	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5077	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5078	}
5079
5080	/* We require EFER to check PAE mode. */
5081	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5082
5083	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5084	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5085	{
5086	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5087	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5088	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5089	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5090	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5091	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5092	}
5093	AssertRCReturn(rc, rc);
5094	}
5095
5096	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5097	}
5098	return rc;
5099	}
5100
5101
5102	/**
5103	* Reads a guest segment register from the current VMCS into the guest-CPU
5104	* context.
5105	*
5106	* @returns VBox status code.
5107	* @param pVCpu Pointer to the VMCPU.
5108	* @param idxSel Index of the selector in the VMCS.
5109	* @param idxLimit Index of the segment limit in the VMCS.
5110	* @param idxBase Index of the segment base in the VMCS.
5111	* @param idxAccess Index of the access rights of the segment in the VMCS.
5112	* @param pSelReg Pointer to the segment selector.
5113	*
5114	* @remarks No-long-jump zone!!!
5115	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5116	* as that takes care of whether to read from the VMCS cache or not.
5117	*/
5118	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5119	PCPUMSELREG pSelReg)
5120	{
5121	uint32_t u32Val = 0;
5122	int rc = VMXReadVmcs32(idxSel, &u32Val);
5123	pSelReg->Sel = (uint16_t)u32Val;
5124	pSelReg->ValidSel = (uint16_t)u32Val;
5125	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5126
5127	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5128	pSelReg->u32Limit = u32Val;
5129
5130	RTGCUINTREG uGCVal = 0;
5131	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5132	pSelReg->u64Base = uGCVal;
5133
5134	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5135	pSelReg->Attr.u = u32Val;
5136	AssertRCReturn(rc, rc);
5137
5138	/*
5139	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5140	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5141	*/
5142	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5143	{
5144	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5145	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5146	}
5147	return rc;
5148	}
5149
5150
5151	/**
5152	* Saves the guest segment registers from the current VMCS into the guest-CPU
5153	* context.
5154	*
5155	* @returns VBox status code.
5156	* @param pVCpu Pointer to the VMCPU.
5157	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5158	* out-of-sync. Make sure to update the required fields
5159	* before using them.
5160	*
5161	* @remarks No-long-jump zone!!!
5162	*/
5163	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5164	{
5165	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5166	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5167	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5168	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5169	#else
5170	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5171	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5172	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5173	#endif
5174
5175	int rc = VINF_SUCCESS;
5176
5177	/* Guest segment registers. */
5178	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5179	{
5180	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5181	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5182	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5183	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5184	rc \|= VMXLOCAL_READ_SEG(ES, es);
5185	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5186	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5187	AssertRCReturn(rc, rc);
5188
5189	/* Restore segment attributes for real-on-v86 mode hack. */
5190	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5191	{
5192	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5193	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5194	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5195	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5196	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5197	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5198	}
5199	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5200	}
5201
5202	/* Guest LDTR. */
5203	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5204	{
5205	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5206	AssertRCReturn(rc, rc);
5207	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5208	}
5209
5210	/* Guest GDTR. */
5211	RTGCUINTREG uGCVal = 0;
5212	uint32_t u32Val = 0;
5213	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5214	{
5215	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5216	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5217	pMixedCtx->gdtr.pGdt = uGCVal;
5218	pMixedCtx->gdtr.cbGdt = u32Val;
5219	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5220	}
5221
5222	/* Guest IDTR. */
5223	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5224	{
5225	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5226	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5227	pMixedCtx->idtr.pIdt = uGCVal;
5228	pMixedCtx->idtr.cbIdt = u32Val;
5229	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5230	}
5231
5232	/* Guest TR. */
5233	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5234	{
5235	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5236
5237	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5238	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5239	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5240	AssertRCReturn(rc, rc);
5241	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5242	}
5243	return rc;
5244	}
5245
5246
5247	/**
5248	* Saves the guest debug registers from the current VMCS into the guest-CPU
5249	* context.
5250	*
5251	* @returns VBox status code.
5252	* @param pVCpu Pointer to the VMCPU.
5253	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5254	* out-of-sync. Make sure to update the required fields
5255	* before using them.
5256	*
5257	* @remarks No-long-jump zone!!!
5258	*/
5259	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5260	{
5261	int rc = VINF_SUCCESS;
5262	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5263	{
5264	RTGCUINTREG uVal;
5265	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5266	pMixedCtx->dr[7] = uVal;
5267
5268	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5269	}
5270	return rc;
5271	}
5272
5273
5274	/**
5275	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5276	*
5277	* @returns VBox status code.
5278	* @param pVCpu Pointer to the VMCPU.
5279	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5280	* out-of-sync. Make sure to update the required fields
5281	* before using them.
5282	*
5283	* @remarks No-long-jump zone!!!
5284	*/
5285	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5286	{
5287	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5288	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5289	return VINF_SUCCESS;
5290	}
5291
5292
5293	/**
5294	* Saves the entire guest state from the currently active VMCS into the
5295	* guest-CPU context. This essentially VMREADs all guest-data.
5296	*
5297	* @returns VBox status code.
5298	* @param pVCpu Pointer to the VMCPU.
5299	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5300	* out-of-sync. Make sure to update the required fields
5301	* before using them.
5302	*/
5303	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5304	{
5305	Assert(pVCpu);
5306	Assert(pMixedCtx);
5307
5308	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5309	return VINF_SUCCESS;
5310
5311	VMMRZCallRing3Disable(pVCpu);
5312	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5313	LogFunc(("\n"));
5314
5315	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5317
5318	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5320
5321	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5322	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5323
5324	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5326
5327	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5329
5330	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5332
5333	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5335
5336	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5338
5339	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5341
5342	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5343	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5344
5345	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5346	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5347
5348	VMMRZCallRing3Enable(pVCpu);
5349	return rc;
5350	}
5351
5352
5353	/**
5354	* Check per-VM and per-VCPU force flag actions that require us to go back to
5355	* ring-3 for one reason or another.
5356	*
5357	* @returns VBox status code (information status code included).
5358	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5359	* ring-3.
5360	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5361	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5362	* interrupts)
5363	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5364	* all EMTs to be in ring-3.
5365	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5366	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5367	* to the EM loop.
5368	*
5369	* @param pVM Pointer to the VM.
5370	* @param pVCpu Pointer to the VMCPU.
5371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5372	* out-of-sync. Make sure to update the required fields
5373	* before using them.
5374	*/
5375	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376	{
5377	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5378
5379	int rc = VERR_INTERNAL_ERROR_5;
5380	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5381	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5382	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5383	{
5384	/* We need the control registers now, make sure the guest-CPU context is updated. */
5385	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5386	AssertRCReturn(rc, rc);
5387
5388	/* Pending HM CR3 sync. */
5389	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5390	{
5391	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5392	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5393	}
5394	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5395	{
5396	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5397	AssertRC(rc);
5398	}
5399
5400	/* Pending PGM C3 sync. */
5401	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5402	{
5403	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5404	if (rc != VINF_SUCCESS)
5405	{
5406	AssertRC(rc);
5407	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5408	return rc;
5409	}
5410	}
5411
5412	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5413	/* -XXX- what was that about single stepping? */
5414	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5415	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5416	{
5417	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5418	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5419	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5420	return rc;
5421	}
5422
5423	/* Pending VM request packets, such as hardware interrupts. */
5424	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5425	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5426	{
5427	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5428	return VINF_EM_PENDING_REQUEST;
5429	}
5430
5431	/* Pending PGM pool flushes. */
5432	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5433	{
5434	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5435	return VINF_PGM_POOL_FLUSH_PENDING;
5436	}
5437
5438	/* Pending DMA requests. */
5439	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5440	{
5441	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5442	return VINF_EM_RAW_TO_R3;
5443	}
5444	}
5445
5446	/* Paranoia. */
5447	Assert(rc != VERR_EM_INTERPRETER);
5448	return VINF_SUCCESS;
5449	}
5450
5451
5452	/**
5453	* Converts any TRPM trap into a pending VMX event. This is typically used when
5454	* entering from ring-3 (not longjmp returns).
5455	*
5456	* @param pVCpu Pointer to the VMCPU.
5457	* @param pCtx Pointer to the guest-CPU context.
5458	*/
5459	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5460	{
5461	if (!TRPMHasTrap(pVCpu))
5462	{
5463	Assert(!pVCpu->hm.s.Event.fPending);
5464	return;
5465	}
5466
5467	uint8_t uVector;
5468	TRPMEVENT enmTrpmEvent;
5469	RTGCUINT uErrCode;
5470	RTGCUINTPTR GCPtrFaultAddress;
5471	uint8_t cbInstr;
5472
5473	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5474	AssertRC(rc);
5475
5476	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5477	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5478	if (enmTrpmEvent == TRPM_TRAP)
5479	{
5480	switch (uVector)
5481	{
5482	case X86_XCPT_BP:
5483	case X86_XCPT_OF:
5484	{
5485	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5486	break;
5487	}
5488
5489	case X86_XCPT_PF:
5490	case X86_XCPT_DF:
5491	case X86_XCPT_TS:
5492	case X86_XCPT_NP:
5493	case X86_XCPT_SS:
5494	case X86_XCPT_GP:
5495	case X86_XCPT_AC:
5496	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5497	/* no break! */
5498	default:
5499	{
5500	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5501	break;
5502	}
5503	}
5504	}
5505	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5506	{
5507	if (uVector != X86_XCPT_NMI)
5508	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5509	else
5510	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5511	}
5512	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5513	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5514	else
5515	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5516
5517	rc = TRPMResetTrap(pVCpu);
5518	AssertRC(rc);
5519	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5520	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5521	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5522	}
5523
5524
5525	/**
5526	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5527	* VT-x to execute any instruction.
5528	*
5529	* @param pvCpu Pointer to the VMCPU.
5530	*/
5531	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5532	{
5533	if (pVCpu->hm.s.Event.fPending)
5534	{
5535	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5536	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5537	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5538	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5539
5540	/* If a trap was already pending, we did something wrong! */
5541	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5542
5543	TRPMEVENT enmTrapType;
5544	switch (uVectorType)
5545	{
5546	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5547	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5548	enmTrapType = TRPM_HARDWARE_INT;
5549	break;
5550	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5551	enmTrapType = TRPM_SOFTWARE_INT;
5552	break;
5553	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5554	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5555	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5556	enmTrapType = TRPM_TRAP;
5557	break;
5558	default:
5559	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5560	enmTrapType = TRPM_32BIT_HACK;
5561	break;
5562	}
5563
5564	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5565	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5566	AssertRC(rc);
5567
5568	if (fErrorCodeValid)
5569	TRPMSetErrorCode(pVCpu, uErrorCode);
5570	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5571	&& uVector == X86_XCPT_PF)
5572	{
5573	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5574	}
5575	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5576	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5577	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5578	{
5579	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5580	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5581	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5582	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5583	}
5584	pVCpu->hm.s.Event.fPending = false;
5585	}
5586	}
5587
5588
5589	/**
5590	* Does the necessary state syncing before doing a longjmp to ring-3.
5591	*
5592	* @param pVM Pointer to the VM.
5593	* @param pVCpu Pointer to the VMCPU.
5594	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5595	* out-of-sync. Make sure to update the required fields
5596	* before using them.
5597	* @param rcExit The reason for exiting to ring-3. Can be
5598	* VINF_VMM_UNKNOWN_RING3_CALL.
5599	*
5600	* @remarks No-long-jmp zone!!!
5601	*/
5602	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5603	{
5604	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5605	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5606
5607	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5608	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5609	AssertRC(rc);
5610
5611	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5612	if (CPUMIsGuestFPUStateActive(pVCpu))
5613	{
5614	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5615	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5616	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5617	}
5618
5619	/* Restore debug registers if necessary and resync on next R0 reentry. */
5620	if (CPUMIsGuestDebugStateActive(pVCpu))
5621	{
5622	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5623	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5624	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5625	}
5626	else if (CPUMIsHyperDebugStateActive(pVCpu))
5627	{
5628	CPUMR0LoadHostDebugState(pVM, pVCpu);
5629	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5630	}
5631
5632	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5633	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5634	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5635	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5636	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5637	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5638	}
5639
5640
5641	/**
5642	* An action requires us to go back to ring-3. This function does the necessary
5643	* steps before we can safely return to ring-3. This is not the same as longjmps
5644	* to ring-3, this is voluntary.
5645	*
5646	* @param pVM Pointer to the VM.
5647	* @param pVCpu Pointer to the VMCPU.
5648	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5649	* out-of-sync. Make sure to update the required fields
5650	* before using them.
5651	* @param rcExit The reason for exiting to ring-3. Can be
5652	* VINF_VMM_UNKNOWN_RING3_CALL.
5653	*/
5654	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5655	{
5656	Assert(pVM);
5657	Assert(pVCpu);
5658	Assert(pMixedCtx);
5659	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5660
5661	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5662	{
5663	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5664	return;
5665	}
5666	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5667	{
5668	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5669	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5670	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5671	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5672	return;
5673	}
5674
5675	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5676	VMMRZCallRing3Disable(pVCpu);
5677	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5678
5679	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5680	hmR0VmxUpdateTRPM(pVCpu);
5681
5682	/* Sync. the guest state. */
5683	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5684	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5685
5686	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5687	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5688	\| CPUM_CHANGED_LDTR
5689	\| CPUM_CHANGED_GDTR
5690	\| CPUM_CHANGED_IDTR
5691	\| CPUM_CHANGED_TR
5692	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5693
5694	/* On our way back from ring-3 the following needs to be done. */
5695	/** @todo This can change with preemption hooks. */
5696	if (rcExit == VINF_EM_RAW_INTERRUPT)
5697	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5698	else
5699	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5700
5701	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5702	VMMRZCallRing3Enable(pVCpu);
5703	}
5704
5705
5706	/**
5707	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5708	* longjump to ring-3 and possibly get preempted.
5709	*
5710	* @param pVCpu Pointer to the VMCPU.
5711	* @param enmOperation The operation causing the ring-3 longjump.
5712	* @param pvUser The user argument (pointer to the possibly
5713	* out-of-date guest-CPU context).
5714	*
5715	* @remarks Must never be called with @a enmOperation ==
5716	* VMMCALLRING3_VM_R0_ASSERTION.
5717	*/
5718	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5719	{
5720	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5721	Assert(pVCpu);
5722	Assert(pvUser);
5723	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5724	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5725
5726	VMMRZCallRing3Disable(pVCpu);
5727	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5728	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5729	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5730	VMMRZCallRing3Enable(pVCpu);
5731	}
5732
5733
5734	/**
5735	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5736	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5737	*
5738	* @returns VBox status code.
5739	* @param pVCpu Pointer to the VMCPU.
5740	*/
5741	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5742	{
5743	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5744	{
5745	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5746	{
5747	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5748	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5749	AssertRC(rc);
5750	}
5751	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5752	}
5753
5754
5755	/**
5756	* Injects any pending events into the guest if the guest is in a state to
5757	* receive them.
5758	*
5759	* @returns VBox status code (informational status codes included).
5760	* @param pVCpu Pointer to the VMCPU.
5761	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5762	* out-of-sync. Make sure to update the required fields
5763	* before using them.
5764	*/
5765	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5766	{
5767	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5768	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5769	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5770	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5771
5772	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5773	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5774	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5775	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5776
5777	int rc = VINF_SUCCESS;
5778	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5779	{
5780	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5781	bool fInject = true;
5782	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5783	{
5784	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5785	AssertRCReturn(rc, rc);
5786	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5787	if ( fBlockInt
5788	\|\| fBlockSti
5789	\|\| fBlockMovSS)
5790	{
5791	fInject = false;
5792	}
5793	}
5794	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5795	&& ( fBlockMovSS
5796	\|\| fBlockSti))
5797	{
5798	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5799	fInject = false;
5800	}
5801
5802	if (fInject)
5803	{
5804	Log(("Injecting pending event\n"));
5805	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5806	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5807	AssertRCReturn(rc, rc);
5808	pVCpu->hm.s.Event.fPending = false;
5809	}
5810	else
5811	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5812	} /** @todo SMI. SMIs take priority over NMIs. */
5813	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5814	{
5815	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5816	if ( !fBlockMovSS
5817	&& !fBlockSti)
5818	{
5819	Log(("Injecting NMI\n"));
5820	RTGCUINTPTR uIntrInfo;
5821	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5822	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5823	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5824	0 /* GCPtrFaultAddress */, &uIntrState);
5825	AssertRCReturn(rc, rc);
5826	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5827	}
5828	else
5829	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5830	}
5831	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5832	{
5833	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5834	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5835	AssertRCReturn(rc, rc);
5836	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5837	if ( !fBlockInt
5838	&& !fBlockSti
5839	&& !fBlockMovSS)
5840	{
5841	uint8_t u8Interrupt;
5842	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5843	if (RT_SUCCESS(rc))
5844	{
5845	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5846	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5847	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5848	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5849	0 /* GCPtrFaultAddress */, &uIntrState);
5850	}
5851	else
5852	{
5853	/** @todo Does this actually happen? If not turn it into an assertion. */
5854	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5855	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5856	rc = VINF_SUCCESS;
5857	}
5858	}
5859	else
5860	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5861	}
5862
5863	/*
5864	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5865	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5866	*/
5867	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5868	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5869	int rc2 = VINF_SUCCESS;
5870	if ( fBlockSti
5871	\|\| fBlockMovSS)
5872	{
5873	if (!DBGFIsStepping(pVCpu))
5874	{
5875	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5876	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5877	{
5878	/*
5879	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5880	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5881	*/
5882	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5883	}
5884	}
5885	else
5886	{
5887	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5888	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5889	uIntrState = 0;
5890	}
5891	}
5892
5893	/*
5894	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5895	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5896	*/
5897	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5898	AssertRC(rc2);
5899
5900	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5901	return rc;
5902	}
5903
5904
5905	/**
5906	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5907	*
5908	* @param pVCpu Pointer to the VMCPU.
5909	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5910	* out-of-sync. Make sure to update the required fields
5911	* before using them.
5912	*/
5913	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5914	{
5915	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5916	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5917	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5918	}
5919
5920
5921	/**
5922	* Injects a double-fault (#DF) exception into the VM.
5923	*
5924	* @returns VBox status code (informational status code included).
5925	* @param pVCpu Pointer to the VMCPU.
5926	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5927	* out-of-sync. Make sure to update the required fields
5928	* before using them.
5929	*/
5930	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5931	{
5932	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5933	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5934	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5935	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5936	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5937	puIntrState);
5938	}
5939
5940
5941	/**
5942	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5943	*
5944	* @param pVCpu Pointer to the VMCPU.
5945	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5946	* out-of-sync. Make sure to update the required fields
5947	* before using them.
5948	*/
5949	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5950	{
5951	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5952	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5953	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5954	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5955	}
5956
5957
5958	/**
5959	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
5960	*
5961	* @param pVCpu Pointer to the VMCPU.
5962	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5963	* out-of-sync. Make sure to update the required fields
5964	* before using them.
5965	* @param cbInstr The value of RIP that is to be pushed on the guest
5966	* stack.
5967	*/
5968	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5969	{
5970	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5971	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5972	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5973	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
5974	}
5975
5976
5977	/**
5978	* Injects a general-protection (#GP) fault into the VM.
5979	*
5980	* @returns VBox status code (informational status code included).
5981	* @param pVCpu Pointer to the VMCPU.
5982	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5983	* out-of-sync. Make sure to update the required fields
5984	* before using them.
5985	* @param u32ErrorCode The error code associated with the #GP.
5986	*/
5987	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
5988	uint32_t *puIntrState)
5989	{
5990	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5991	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5992	if (fErrorCodeValid)
5993	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5994	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5995	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
5996	puIntrState);
5997	}
5998
5999
6000	/**
6001	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6002	*
6003	* @param pVCpu Pointer to the VMCPU.
6004	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6005	* out-of-sync. Make sure to update the required fields
6006	* before using them.
6007	* @param uVector The software interrupt vector number.
6008	* @param cbInstr The value of RIP that is to be pushed on the guest
6009	* stack.
6010	*/
6011	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6012	{
6013	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6014	if ( uVector == X86_XCPT_BP
6015	\|\| uVector == X86_XCPT_OF)
6016	{
6017	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6018	}
6019	else
6020	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6021	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6022	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6023	}
6024
6025
6026	/**
6027	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6028	* stack.
6029	*
6030	* @returns VBox status code (information status code included).
6031	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6032	* @param pVM Pointer to the VM.
6033	* @param pMixedCtx Pointer to the guest-CPU context.
6034	* @param uValue The value to push to the guest stack.
6035	*/
6036	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6037	{
6038	/*
6039	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6040	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6041	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6042	*/
6043	if (pMixedCtx->sp == 1)
6044	return VINF_EM_RESET;
6045	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6046	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6047	AssertRCReturn(rc, rc);
6048	return rc;
6049	}
6050
6051
6052	/**
6053	* Injects an event into the guest upon VM-entry by updating the relevant fields
6054	* in the VM-entry area in the VMCS.
6055	*
6056	* @returns VBox status code (informational error codes included).
6057	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6058	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6059	*
6060	* @param pVCpu Pointer to the VMCPU.
6061	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6062	* be out-of-sync. Make sure to update the required
6063	* fields before using them.
6064	* @param u64IntrInfo The VM-entry interruption-information field.
6065	* @param cbInstr The VM-entry instruction length in bytes (for
6066	* software interrupts, exceptions and privileged
6067	* software exceptions).
6068	* @param u32ErrCode The VM-entry exception error code.
6069	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6070	* @param puIntrState Pointer to the current guest interruptibility-state.
6071	* This interruptibility-state will be updated if
6072	* necessary. This cannot not be NULL.
6073	*
6074	* @remarks No-long-jump zone!!!
6075	* @remarks Requires CR0!
6076	*/
6077	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6078	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6079	{
6080	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6081	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6082	Assert(puIntrState);
6083	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6084
6085	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6086	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6087
6088	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6089	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6090	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6091
6092	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6093
6094	/* We require CR0 to check if the guest is in real-mode. */
6095	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6096	AssertRCReturn(rc, rc);
6097
6098	/*
6099	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6100	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6101	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6102	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6103	*/
6104	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6105	{
6106	PVM pVM = pVCpu->CTX_SUFF(pVM);
6107	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6108	{
6109	Assert(PDMVmmDevHeapIsEnabled(pVM));
6110	Assert(pVM->hm.s.vmx.pRealModeTSS);
6111
6112	/* Save the required guest state bits from the VMCS. */
6113	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6114	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6115	AssertRCReturn(rc, rc);
6116
6117	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6118	const size_t cbIdtEntry = 4;
6119	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6120	{
6121	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6122	if (uVector == X86_XCPT_DF)
6123	return VINF_EM_RESET;
6124	else if (uVector == X86_XCPT_GP)
6125	{
6126	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6127	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6128	}
6129
6130	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6131	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6132	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6133	}
6134
6135	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6136	uint16_t uGuestIp = pMixedCtx->ip;
6137	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6138	{
6139	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6140	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6141	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6142	}
6143	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6144	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6145
6146	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6147	uint16_t offIdtEntry = 0;
6148	RTSEL selIdtEntry = 0;
6149	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6150	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6151	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6152	AssertRCReturn(rc, rc);
6153
6154	/* Construct the stack frame for the interrupt/exception handler. */
6155	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6156	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6157	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6158	AssertRCReturn(rc, rc);
6159
6160	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6161	if (rc == VINF_SUCCESS)
6162	{
6163	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6164	pMixedCtx->rip = offIdtEntry;
6165	pMixedCtx->cs.Sel = selIdtEntry;
6166	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6167	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6168	&& uVector == X86_XCPT_PF)
6169	{
6170	pMixedCtx->cr2 = GCPtrFaultAddress;
6171	}
6172	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6173	\| HM_CHANGED_GUEST_RIP
6174	\| HM_CHANGED_GUEST_RFLAGS
6175	\| HM_CHANGED_GUEST_RSP;
6176
6177	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6178	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6179	{
6180	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6181	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6182	Log(("Clearing inhibition due to STI.\n"));
6183	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6184	}
6185	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6186	}
6187	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6188	return rc;
6189	}
6190	else
6191	{
6192	/*
6193	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6194	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6195	*/
6196	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6197	}
6198	}
6199
6200	/* Validate. */
6201	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6202	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6203	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6204
6205	/* Inject. */
6206	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6207	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6208	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6209	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6210
6211	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6212	&& uVector == X86_XCPT_PF)
6213	{
6214	pMixedCtx->cr2 = GCPtrFaultAddress;
6215	}
6216	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6217
6218	AssertRCReturn(rc, rc);
6219	return rc;
6220	}
6221
6222
6223	/**
6224	* Enters the VT-x session.
6225	*
6226	* @returns VBox status code.
6227	* @param pVM Pointer to the VM.
6228	* @param pVCpu Pointer to the VMCPU.
6229	* @param pCpu Pointer to the CPU info struct.
6230	*/
6231	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6232	{
6233	AssertPtr(pVM);
6234	AssertPtr(pVCpu);
6235	Assert(pVM->hm.s.vmx.fSupported);
6236	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6237	NOREF(pCpu);
6238
6239	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6240
6241	/* Make sure we're in VMX root mode. */
6242	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6243	if (!(u32HostCR4 & X86_CR4_VMXE))
6244	{
6245	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6246	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6247	}
6248
6249	/* Load the active VMCS as the current one. */
6250	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6251	if (RT_FAILURE(rc))
6252	return rc;
6253
6254	/** @todo this will change with preemption hooks where can can VMRESUME as long
6255	* as we're no preempted. */
6256	pVCpu->hm.s.fResumeVM = false;
6257	return VINF_SUCCESS;
6258	}
6259
6260
6261	/**
6262	* Leaves the VT-x session.
6263	*
6264	* @returns VBox status code.
6265	* @param pVM Pointer to the VM.
6266	* @param pVCpu Pointer to the VMCPU.
6267	* @param pCtx Pointer to the guest-CPU context.
6268	*/
6269	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6270	{
6271	AssertPtr(pVCpu);
6272	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6273	NOREF(pVM);
6274	NOREF(pCtx);
6275
6276	/** @todo this will change with preemption hooks where we only VMCLEAR when
6277	* we are actually going to be preempted, not all the time like we
6278	* currently do. */
6279	/*
6280	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6281	* and mark the VMCS launch-state as "clear".
6282	*/
6283	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6284	return rc;
6285	}
6286
6287
6288	/**
6289	* Saves the host state in the VMCS host-state.
6290	* Sets up the VM-exit MSR-load area.
6291	*
6292	* The CPU state will be loaded from these fields on every successful VM-exit.
6293	*
6294	* @returns VBox status code.
6295	* @param pVM Pointer to the VM.
6296	* @param pVCpu Pointer to the VMCPU.
6297	*
6298	* @remarks No-long-jump zone!!!
6299	*/
6300	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6301	{
6302	AssertPtr(pVM);
6303	AssertPtr(pVCpu);
6304	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6305
6306	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6307
6308	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6309	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6310	return VINF_SUCCESS;
6311
6312	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6313	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6314
6315	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6316	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6317
6318	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6319	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6320
6321	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6322	return rc;
6323	}
6324
6325
6326	/**
6327	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6328	* loaded from these fields on every successful VM-entry.
6329	*
6330	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6331	* Sets up the VM-entry controls.
6332	* Sets up the appropriate VMX non-root function to execute guest code based on
6333	* the guest CPU mode.
6334	*
6335	* @returns VBox status code.
6336	* @param pVM Pointer to the VM.
6337	* @param pVCpu Pointer to the VMCPU.
6338	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6339	* out-of-sync. Make sure to update the required fields
6340	* before using them.
6341	*
6342	* @remarks No-long-jump zone!!!
6343	*/
6344	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6345	{
6346	AssertPtr(pVM);
6347	AssertPtr(pVCpu);
6348	AssertPtr(pMixedCtx);
6349	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6350
6351	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6352
6353	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6354
6355	/* Determine real-on-v86 mode. */
6356	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6357	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6358	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6359	{
6360	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6361	}
6362
6363	/** @todo if the order of loading is important, inform it via comments here */
6364	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6365	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6366
6367	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6368	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6369
6370	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6371	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6372
6373	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6374	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6375
6376	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6377	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6378
6379	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6380	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6381
6382	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6383	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6384
6385	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6386	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6387
6388	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6389	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6390
6391	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6392	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6393
6394	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6395	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6396	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6397
6398	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6399	return rc;
6400	}
6401
6402
6403	/**
6404	* Does the preparations before executing guest code in VT-x.
6405	*
6406	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6407	* recompiler. We must be cautious what we do here regarding committing
6408	* guest-state information into the the VMCS assuming we assuredly execute the
6409	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6410	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6411	* that the recompiler can (and should) use them when it resumes guest
6412	* execution. Otherwise such operations must be done when we can no longer
6413	* exit to ring-3.
6414	*
6415	* @returns VBox status code (informational status codes included).
6416	* @retval VINF_SUCCESS if we can proceed with running the guest.
6417	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6418	* into the guest.
6419	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6420	*
6421	* @param pVM Pointer to the VM.
6422	* @param pVCpu Pointer to the VMCPU.
6423	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6424	* out-of-sync. Make sure to update the required fields
6425	* before using them.
6426	* @param pVmxTransient Pointer to the VMX transient structure.
6427	*
6428	* @remarks Called with preemption disabled.
6429	*/
6430	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6431	{
6432	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6433
6434	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6435	PGMRZDynMapFlushAutoSet(pVCpu);
6436	#endif
6437
6438	/* Check force flag actions that might require us to go back to ring-3. */
6439	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6440	if (rc != VINF_SUCCESS)
6441	return rc;
6442
6443	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6444	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6445	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6446	{
6447	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6448	RTGCPHYS GCPhysApicBase;
6449	GCPhysApicBase = pMixedCtx->msrApicBase;
6450	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6451
6452	/* Unalias any existing mapping. */
6453	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6454	AssertRCReturn(rc, rc);
6455
6456	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6457	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6458	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6459	AssertRCReturn(rc, rc);
6460
6461	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6462	}
6463
6464	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6465	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6466	pVmxTransient->uEFlags = ASMIntDisableFlags();
6467	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6468	{
6469	ASMSetFlags(pVmxTransient->uEFlags);
6470	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6471	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6472	return VINF_EM_RAW_INTERRUPT;
6473	}
6474	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6475	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6476	#endif
6477
6478	/*
6479	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6480	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6481	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6482	*/
6483	/** @todo r=bird: You reverse the effect of calling PDMGetInterrupt by
6484	* handing it over to TPRM like we do in REMR3StateBack using
6485	* TRPMAssertTrap and the other setters. */
6486	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6487	AssertRCReturn(rc, rc);
6488	return rc;
6489	}
6490
6491
6492	/**
6493	* Prepares to run guest code in VT-x and we've committed to doing so. This
6494	* means there is no backing out to ring-3 or anywhere else at this
6495	* point.
6496	*
6497	* @param pVM Pointer to the VM.
6498	* @param pVCpu Pointer to the VMCPU.
6499	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6500	* out-of-sync. Make sure to update the required fields
6501	* before using them.
6502	* @param pVmxTransient Pointer to the VMX transient structure.
6503	*
6504	* @remarks Called with preemption disabled.
6505	* @remarks No-long-jump zone!!!
6506	*/
6507	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6508	{
6509	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6510	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6511
6512	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6513	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6514	pVmxTransient->uEFlags = ASMIntDisableFlags();
6515	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6516	#endif
6517
6518	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6519	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6520	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6521	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6522	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6523	#endif
6524	int rc = VINF_SUCCESS;
6525	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6526	{
6527	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6528	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6529	}
6530	else if (pVCpu->hm.s.fContextUseFlags)
6531	{
6532	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6533	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6534	}
6535	AssertRC(rc);
6536	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6537
6538	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6539	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6540	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6541
6542	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6543	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6544	{
6545	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6546	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6547	}
6548
6549	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6550	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6551	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6552
6553	/*
6554	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6555	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6556	*/
6557	if (pVM->hm.s.fTPRPatchingActive)
6558	{
6559	Assert(!CPUMIsGuestInLongMode(pVCpu));
6560
6561	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6562	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6563	AssertRC(rc);
6564
6565	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6566	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6567	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6568	see hmR0VmxLoadGuestApicState(). */
6569	}
6570
6571	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6572	/*
6573	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6574	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6575	*/
6576	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6577	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6578	{
6579	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6580	uint64_t u64HostTscAux = 0;
6581	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6582	AssertRC(rc2);
6583	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6584	}
6585	#endif
6586
6587	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6588	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6589	to start executing. */
6590	}
6591
6592
6593	/**
6594	* Performs some essential restoration of state after running guest code in
6595	* VT-x.
6596	*
6597	* @param pVM Pointer to the VM.
6598	* @param pVCpu Pointer to the VMCPU.
6599	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6600	* out-of-sync. Make sure to update the required fields
6601	* before using them.
6602	* @param pVmxTransient Pointer to the VMX transient structure.
6603	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6604	*
6605	* @remarks Called with interrupts disabled.
6606	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6607	* unconditionally when it is safe to do so.
6608	*/
6609	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6610	{
6611	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6612	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6613
6614	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6615	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6616	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6617	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6618	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6619
6620	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6621	{
6622	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6623	/* Restore host's TSC_AUX. */
6624	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6625	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6626	#endif
6627	/** @todo Find a way to fix hardcoding a guestimate. */
6628	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6629	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6630	}
6631
6632	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6633	Assert(!(ASMGetFlags() & X86_EFL_IF));
6634	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6635
6636	/* Restore the effects of TPR patching if any. */
6637	if (pVM->hm.s.fTPRPatchingActive)
6638	{
6639	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6640	AssertRC(rc);
6641	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6642	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6643	}
6644
6645	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6646	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6647
6648	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6649	uint32_t uExitReason;
6650	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6651	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6652	AssertRC(rc);
6653	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6654	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6655
6656	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6657	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6658
6659	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6660	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6661	{
6662	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6663	return;
6664	}
6665
6666	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6667	{
6668	/* Update the guest interruptibility-state from the VMCS. */
6669	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6670	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6671	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6672	AssertRC(rc);
6673	#endif
6674	/*
6675	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6676	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6677	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6678	*/
6679	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6680	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6681	{
6682	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6683	AssertRC(rc);
6684	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6685	}
6686	}
6687	}
6688
6689
6690	/**
6691	* Runs the guest code using VT-x.
6692	*
6693	* @returns VBox status code.
6694	* @param pVM Pointer to the VM.
6695	* @param pVCpu Pointer to the VMCPU.
6696	* @param pCtx Pointer to the guest-CPU context.
6697	*
6698	* @remarks Called with preemption disabled.
6699	*/
6700	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6701	{
6702	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6703	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6704
6705	VMXTRANSIENT VmxTransient;
6706	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6707	int rc = VERR_INTERNAL_ERROR_5;
6708	uint32_t cLoops = 0;
6709	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6710
6711	for (;; cLoops++)
6712	{
6713	Assert(!HMR0SuspendPending());
6714	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6715	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6716	(unsigned)RTMpCpuId(), cLoops));
6717
6718	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6719	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6720	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6721	if (rc != VINF_SUCCESS)
6722	break;
6723
6724	/*
6725	* No longjmps to ring-3 from this point on!!!
6726	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6727	* This also disables flushing of the R0-logger instance (if any).
6728	*/
6729	VMMRZCallRing3Disable(pVCpu);
6730	VMMRZCallRing3RemoveNotification(pVCpu);
6731	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6732
6733	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6734	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6735
6736	/*
6737	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6738	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6739	*/
6740	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6741	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6742	{
6743	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6744	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6745	return rc;
6746	}
6747
6748	/* Handle the VM-exit. */
6749	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6750	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6751	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6752	#ifdef HM_PROFILE_EXIT_DISPATCH
6753	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6754	#endif
6755	#ifdef HMVMX_USE_FUNCTION_TABLE
6756	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6757	#else
6758	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6759	#endif
6760	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6761	if (rc != VINF_SUCCESS)
6762	break;
6763	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6764	{
6765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6766	rc = VINF_EM_RAW_INTERRUPT;
6767	break;
6768	}
6769	}
6770
6771	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6772	if (rc == VERR_EM_INTERPRETER)
6773	rc = VINF_EM_RAW_EMULATE_INSTR;
6774	else if (rc == VINF_EM_RESET)
6775	rc = VINF_EM_TRIPLE_FAULT;
6776	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6777	return rc;
6778	}
6779
6780
6781	#ifndef HMVMX_USE_FUNCTION_TABLE
6782	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6783	{
6784	int rc;
6785	switch (rcReason)
6786	{
6787	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6788	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6789	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6790	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6791	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6792	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6793	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6794	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6795	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6796	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6797	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6798	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6799	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6800	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6801	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6802	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6803	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6804	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6805	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6806	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6807	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6808	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6809	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6811	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6812	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6813	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6814	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6815	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6820
6821	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6830
6831	case VMX_EXIT_VMCALL:
6832	case VMX_EXIT_VMCLEAR:
6833	case VMX_EXIT_VMLAUNCH:
6834	case VMX_EXIT_VMPTRLD:
6835	case VMX_EXIT_VMPTRST:
6836	case VMX_EXIT_VMREAD:
6837	case VMX_EXIT_VMRESUME:
6838	case VMX_EXIT_VMWRITE:
6839	case VMX_EXIT_VMXOFF:
6840	case VMX_EXIT_VMXON:
6841	case VMX_EXIT_INVEPT:
6842	case VMX_EXIT_INVVPID:
6843	case VMX_EXIT_VMFUNC:
6844	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6845	break;
6846	default:
6847	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6848	break;
6849	}
6850	return rc;
6851	}
6852	#endif
6853
6854
6855	/** Profiling macro. */
6856	#ifdef HM_PROFILE_EXIT_DISPATCH
6857	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6858	#else
6859	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6860	#endif
6861
6862
6863	#ifdef DEBUG
6864	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6865	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6866	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6867	# define VMX_ASSERT_PREEMPT_CPUID() \
6868	do \
6869	{ \
6870	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6871	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6872	} while (0)
6873
6874	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6875	do { \
6876	AssertPtr(pVCpu); \
6877	AssertPtr(pMixedCtx); \
6878	AssertPtr(pVmxTransient); \
6879	Assert(pVmxTransient->fVMEntryFailed == false); \
6880	Assert(ASMIntAreEnabled()); \
6881	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6882	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6883	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6884	(unsigned)pVCpu->idCpu)); \
6885	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6886	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6887	VMX_ASSERT_PREEMPT_CPUID(); \
6888	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6889	} while (0)
6890	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6891	do { \
6892	LogFunc(("\n")); \
6893	} while(0)
6894	#else /* Release builds */
6895	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6896	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6897	#endif
6898
6899
6900	/**
6901	* Advances the guest RIP after reading it from the VMCS.
6902	*
6903	* @returns VBox status code.
6904	* @param pVCpu Pointer to the VMCPU.
6905	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6906	* out-of-sync. Make sure to update the required fields
6907	* before using them.
6908	* @param pVmxTransient Pointer to the VMX transient structure.
6909	*
6910	* @remarks No-long-jump zone!!!
6911	*/
6912	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6913	{
6914	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6915	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6916	AssertRCReturn(rc, rc);
6917
6918	pMixedCtx->rip += pVmxTransient->cbInstr;
6919	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6920	return rc;
6921	}
6922
6923
6924	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6925	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6926	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6927	/**
6928	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6929	*/
6930	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6931	{
6932	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6933	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6934	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6935	Assert(ASMIntAreEnabled());
6936	return VINF_SUCCESS;
6937	#else
6938	return VINF_EM_RAW_INTERRUPT;
6939	#endif
6940	}
6941
6942
6943	/**
6944	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6945	*/
6946	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6947	{
6948	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6949	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6950
6951	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6952	AssertRCReturn(rc, rc);
6953
6954	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6955	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6956	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6957
6958	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6959	{
6960	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6961	return VINF_EM_RAW_INTERRUPT;
6962	}
6963
6964	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6965	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6966	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6967	{
6968	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6969	return VINF_SUCCESS;
6970	}
6971	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6972	{
6973	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6974	return rc;
6975	}
6976
6977	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6978	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6979	switch (uIntrType)
6980	{
6981	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6982	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6983	/* no break */
6984	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
6985	{
6986	switch (uVector)
6987	{
6988	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6989	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6990	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6991	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6992	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6993	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
6994	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6995	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6996	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6997	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6998	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6999	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7000	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7001	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7002	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7003	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7004	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7005	#endif
7006	default:
7007	{
7008	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7009	AssertRCReturn(rc, rc);
7010
7011	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7012	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7013	{
7014	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7015	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7016	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7017	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7018	AssertRCReturn(rc, rc);
7019	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7020	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7021	0 /* GCPtrFaultAddress */);
7022	AssertRCReturn(rc, rc);
7023	}
7024	else
7025	{
7026	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7027	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7028	}
7029	break;
7030	}
7031	}
7032	break;
7033	}
7034
7035	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7036	default:
7037	{
7038	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7039	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7040	break;
7041	}
7042	}
7043	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7044	return rc;
7045	}
7046
7047
7048	/**
7049	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7050	*/
7051	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7052	{
7053	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7054
7055	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7056	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7057	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7058	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7059	AssertRCReturn(rc, rc);
7060
7061	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7062	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7063	return VINF_SUCCESS;
7064	}
7065
7066
7067	/**
7068	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7069	*/
7070	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7071	{
7072	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7073	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7074	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7075	}
7076
7077
7078	/**
7079	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7080	*/
7081	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7082	{
7083	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7085	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7086	}
7087
7088
7089	/**
7090	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7091	*/
7092	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7093	{
7094	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7095	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7096	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7097	}
7098
7099
7100	/**
7101	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7102	*/
7103	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7104	{
7105	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7106	PVM pVM = pVCpu->CTX_SUFF(pVM);
7107	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7108	if (RT_LIKELY(rc == VINF_SUCCESS))
7109	{
7110	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7111	Assert(pVmxTransient->cbInstr == 2);
7112	}
7113	else
7114	{
7115	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7116	rc = VERR_EM_INTERPRETER;
7117	}
7118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7119	return rc;
7120	}
7121
7122
7123	/**
7124	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7125	*/
7126	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7127	{
7128	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7129	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7130	AssertRCReturn(rc, rc);
7131
7132	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7133	return VINF_EM_RAW_EMULATE_INSTR;
7134
7135	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7136	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7137	}
7138
7139
7140	/**
7141	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7142	*/
7143	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7144	{
7145	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7146	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7147	AssertRCReturn(rc, rc);
7148
7149	PVM pVM = pVCpu->CTX_SUFF(pVM);
7150	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7151	if (RT_LIKELY(rc == VINF_SUCCESS))
7152	{
7153	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7154	Assert(pVmxTransient->cbInstr == 2);
7155	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7156	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7157	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7158	}
7159	else
7160	{
7161	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7162	rc = VERR_EM_INTERPRETER;
7163	}
7164	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7165	return rc;
7166	}
7167
7168
7169	/**
7170	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7171	*/
7172	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7173	{
7174	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7175	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7176	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7177	AssertRCReturn(rc, rc);
7178
7179	PVM pVM = pVCpu->CTX_SUFF(pVM);
7180	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7181	if (RT_LIKELY(rc == VINF_SUCCESS))
7182	{
7183	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7184	Assert(pVmxTransient->cbInstr == 3);
7185	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7186	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7187	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7188	}
7189	else
7190	{
7191	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7192	rc = VERR_EM_INTERPRETER;
7193	}
7194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7195	return rc;
7196	}
7197
7198
7199	/**
7200	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7201	*/
7202	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7203	{
7204	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7205	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7206	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7207	AssertRCReturn(rc, rc);
7208
7209	PVM pVM = pVCpu->CTX_SUFF(pVM);
7210	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7211	if (RT_LIKELY(rc == VINF_SUCCESS))
7212	{
7213	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7214	Assert(pVmxTransient->cbInstr == 2);
7215	}
7216	else
7217	{
7218	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7219	rc = VERR_EM_INTERPRETER;
7220	}
7221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7222	return rc;
7223	}
7224
7225
7226	/**
7227	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7228	*/
7229	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7230	{
7231	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7232	PVM pVM = pVCpu->CTX_SUFF(pVM);
7233	Assert(!pVM->hm.s.fNestedPaging);
7234
7235	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7236	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7237	AssertRCReturn(rc, rc);
7238
7239	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7240	rc = VBOXSTRICTRC_VAL(rc2);
7241	if (RT_LIKELY(rc == VINF_SUCCESS))
7242	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7243	else
7244	{
7245	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7246	pVmxTransient->uExitQualification, rc));
7247	}
7248	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7249	return rc;
7250	}
7251
7252
7253	/**
7254	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7255	*/
7256	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7257	{
7258	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7259	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7260	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7261	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7262	AssertRCReturn(rc, rc);
7263
7264	PVM pVM = pVCpu->CTX_SUFF(pVM);
7265	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7266	if (RT_LIKELY(rc == VINF_SUCCESS))
7267	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7268	else
7269	{
7270	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7271	rc = VERR_EM_INTERPRETER;
7272	}
7273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7274	return rc;
7275	}
7276
7277
7278	/**
7279	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7280	*/
7281	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7282	{
7283	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7284	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7285	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7286	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7287	AssertRCReturn(rc, rc);
7288
7289	PVM pVM = pVCpu->CTX_SUFF(pVM);
7290	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7291	rc = VBOXSTRICTRC_VAL(rc2);
7292	if (RT_LIKELY( rc == VINF_SUCCESS
7293	\|\| rc == VINF_EM_HALT))
7294	{
7295	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7296	AssertRCReturn(rc3, rc3);
7297
7298	if ( rc == VINF_EM_HALT
7299	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7300	{
7301	rc = VINF_SUCCESS;
7302	}
7303	}
7304	else
7305	{
7306	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7307	rc = VERR_EM_INTERPRETER;
7308	}
7309	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7310	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7311	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7312	return rc;
7313	}
7314
7315
7316	/**
7317	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7318	*/
7319	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7320	{
7321	/*
7322	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7323	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7324	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7325	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7326	*/
7327	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7328	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7329	}
7330
7331
7332	/**
7333	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7334	*/
7335	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7336	{
7337	/*
7338	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7339	* root operation. If we get there there is something funny going on.
7340	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7341	*/
7342	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7343	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7344	}
7345
7346
7347	/**
7348	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7349	*/
7350	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7351	{
7352	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7353	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7354	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7355	}
7356
7357
7358	/**
7359	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7360	*/
7361	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7362	{
7363	/*
7364	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7365	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7366	* See Intel spec. 25.3 "Other Causes of VM-exits".
7367	*/
7368	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7369	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7370	}
7371
7372
7373	/**
7374	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7375	* VM-exit.
7376	*/
7377	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7378	{
7379	/*
7380	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7381	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7382	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7383	*/
7384	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7385	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7386	}
7387
7388
7389	/**
7390	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7391	* VM-exit.
7392	*/
7393	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7394	{
7395	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7396	return VINF_EM_RESET;
7397	}
7398
7399
7400	/**
7401	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7402	*/
7403	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7404	{
7405	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7406	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7407	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7408	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7409	AssertRCReturn(rc, rc);
7410
7411	pMixedCtx->rip++;
7412	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7413	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7414	rc = VINF_SUCCESS;
7415	else
7416	rc = VINF_EM_HALT;
7417
7418	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7419	return rc;
7420	}
7421
7422
7423	/**
7424	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7425	*/
7426	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7427	{
7428	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7429	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7430	return VINF_SUCCESS;
7431	}
7432
7433
7434	/**
7435	* VM-exit handler for expiry of the VMX preemption timer.
7436	*/
7437	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7438	{
7439	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7440
7441	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7442	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7443
7444	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7445	PVM pVM = pVCpu->CTX_SUFF(pVM);
7446	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7447	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7448	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7449	}
7450
7451
7452	/**
7453	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7454	*/
7455	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7456	{
7457	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7458	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7459	/** @todo check if XSETBV is supported by the recompiler. */
7460	return VERR_EM_INTERPRETER;
7461	}
7462
7463
7464	/**
7465	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7466	*/
7467	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7468	{
7469	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7470	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7471	/** @todo implement EMInterpretInvpcid() */
7472	return VERR_EM_INTERPRETER;
7473	}
7474
7475
7476	/**
7477	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7478	* Error VM-exit.
7479	*/
7480	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7481	{
7482	uint32_t uIntrState;
7483	HMVMXHCUINTREG uHCReg;
7484	uint64_t u64Val;
7485	uint32_t u32Val;
7486
7487	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7488	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7489	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7490	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7491	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7492	AssertRCReturn(rc, rc);
7493
7494	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7495	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7496	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7497	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7498
7499	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7500	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7501	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7502	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7503	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7504	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7505	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7506	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7507	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7508	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7509	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7510	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7511
7512	PVM pVM = pVCpu->CTX_SUFF(pVM);
7513	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7514
7515	return VERR_VMX_INVALID_GUEST_STATE;
7516	}
7517
7518
7519	/**
7520	* VM-exit handler for VM-entry failure due to an MSR-load
7521	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7522	*/
7523	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7524	{
7525	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7526	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7527	}
7528
7529
7530	/**
7531	* VM-exit handler for VM-entry failure due to a machine-check event
7532	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7533	*/
7534	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7535	{
7536	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7537	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7538	}
7539
7540
7541	/**
7542	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7543	* theory.
7544	*/
7545	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7546	{
7547	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7548	return VERR_VMX_UNDEFINED_EXIT_CODE;
7549	}
7550
7551
7552	/**
7553	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7554	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7555	* Conditional VM-exit.
7556	*/
7557	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7558	{
7559	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7560	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7562	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7563	return VERR_EM_INTERPRETER;
7564	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7565	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7566	}
7567
7568
7569	/**
7570	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7571	*/
7572	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7573	{
7574	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7575	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7576	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7577	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7578	return VERR_EM_INTERPRETER;
7579	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7580	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7581	}
7582
7583
7584	/**
7585	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7586	*/
7587	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7588	{
7589	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7590	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7591	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7592	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7593	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7594	AssertRCReturn(rc, rc);
7595
7596	PVM pVM = pVCpu->CTX_SUFF(pVM);
7597	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7598	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7599	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7601
7602	if (RT_LIKELY(rc == VINF_SUCCESS))
7603	{
7604	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7605	Assert(pVmxTransient->cbInstr == 2);
7606	}
7607	return rc;
7608	}
7609
7610
7611	/**
7612	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7613	*/
7614	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7615	{
7616	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7617	PVM pVM = pVCpu->CTX_SUFF(pVM);
7618	int rc = VINF_SUCCESS;
7619
7620	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7621	if ( pVM->hm.s.fTPRPatchingActive
7622	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7623	{
7624	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7625	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7626	{
7627	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7628	AssertRC(rc);
7629	}
7630
7631	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7632	Assert(pVmxTransient->cbInstr == 2);
7633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7634	return VINF_SUCCESS;
7635	}
7636
7637	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7638	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7639	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7640	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7641	AssertRCReturn(rc, rc);
7642	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7643
7644	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7645	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7646	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7647
7648	if (RT_LIKELY(rc == VINF_SUCCESS))
7649	{
7650	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7651
7652	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7653	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7654	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7655	{
7656	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7657	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7658	}
7659	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7660	{
7661	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7662	AssertRCReturn(rc, rc);
7663	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7664	}
7665	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7666	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7667
7668	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7669	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7670	{
7671	switch (pMixedCtx->ecx)
7672	{
7673	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7674	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7675	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7676	case MSR_K8_FS_BASE: /* no break */
7677	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7678	/* MSR_K8_KERNEL_GS_BASE: Nothing to do as it's not part of the VMCS. Manually loaded each time on VM-entry. */
7679	}
7680	}
7681	#ifdef VBOX_STRICT
7682	else
7683	{
7684	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7685	switch (pMixedCtx->ecx)
7686	{
7687	case MSR_IA32_SYSENTER_CS:
7688	case MSR_IA32_SYSENTER_EIP:
7689	case MSR_IA32_SYSENTER_ESP:
7690	case MSR_K8_FS_BASE:
7691	case MSR_K8_GS_BASE:
7692	{
7693	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7694	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7695	}
7696
7697	case MSR_K8_LSTAR:
7698	case MSR_K6_STAR:
7699	case MSR_K8_SF_MASK:
7700	case MSR_K8_TSC_AUX:
7701	case MSR_K8_KERNEL_GS_BASE:
7702	{
7703	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7704	pMixedCtx->ecx));
7705	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7706	}
7707	}
7708	}
7709	#endif /* VBOX_STRICT */
7710	}
7711	return rc;
7712	}
7713
7714
7715	/**
7716	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7717	*/
7718	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7719	{
7720	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7721	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7722	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7723	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7724	return VERR_EM_INTERPRETER;
7725	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7726	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7727	}
7728
7729
7730	/**
7731	* VM-exit handler for when the TPR value is lowered below the specified
7732	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7733	*/
7734	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7735	{
7736	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7737	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7738
7739	/*
7740	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7741	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7742	* resume guest execution.
7743	*/
7744	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7745	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7746	return VINF_SUCCESS;
7747	}
7748
7749
7750	/**
7751	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7752	* VM-exit.
7753	*
7754	* @retval VINF_SUCCESS when guest execution can continue.
7755	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7756	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7757	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7758	* recompiler.
7759	*/
7760	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7761	{
7762	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7763	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7764	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7765	AssertRCReturn(rc, rc);
7766
7767	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7768	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7769	PVM pVM = pVCpu->CTX_SUFF(pVM);
7770	switch (uAccessType)
7771	{
7772	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7773	{
7774	#if 0
7775	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7776	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7777	#else
7778	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7779	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7780	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7781	#endif
7782	AssertRCReturn(rc, rc);
7783
7784	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7785	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7786	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7787	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7788
7789	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7790	{
7791	case 0: /* CR0 */
7792	Log(("CRX CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7793	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7794	break;
7795	case 2: /* C2 **/
7796	/* Nothing to do here, CR2 it's not part of the VMCS. */
7797	break;
7798	case 3: /* CR3 */
7799	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7800	Log(("CRX CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7801	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7802	break;
7803	case 4: /* CR4 */
7804	Log(("CRX CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7805	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7806	break;
7807	case 8: /* CR8 */
7808	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7809	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7810	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7811	break;
7812	default:
7813	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7814	break;
7815	}
7816
7817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7818	break;
7819	}
7820
7821	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7822	{
7823	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7824	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7825	AssertRCReturn(rc, rc);
7826	Assert( !pVM->hm.s.fNestedPaging
7827	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7828	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7829
7830	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7831	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7832	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7833
7834	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7835	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7836	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7837	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7838	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7839	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7840	break;
7841	}
7842
7843	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7844	{
7845	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7846	AssertRCReturn(rc, rc);
7847	rc = EMInterpretCLTS(pVM, pVCpu);
7848	AssertRCReturn(rc, rc);
7849	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7851	Log(("CRX CLTS write rc=%d\n", rc));
7852	break;
7853	}
7854
7855	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7856	{
7857	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7858	AssertRCReturn(rc, rc);
7859	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7860	if (RT_LIKELY(rc == VINF_SUCCESS))
7861	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7863	Log(("CRX LMSW write rc=%d\n", rc));
7864	break;
7865	}
7866
7867	default:
7868	{
7869	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7870	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7871	}
7872	}
7873
7874	/* Validate possible error codes. */
7875	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7876	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7877	if (RT_SUCCESS(rc))
7878	{
7879	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7880	AssertRCReturn(rc2, rc2);
7881	}
7882
7883	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7884	return rc;
7885	}
7886
7887
7888	/**
7889	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7890	* VM-exit.
7891	*/
7892	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7893	{
7894	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7895	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7896
7897	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7898	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7899	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7900	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7901	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7902	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7903	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7904	AssertRCReturn(rc, rc);
7905	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7906
7907	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7908	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7909	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7910	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7911	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7912	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7913	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7914
7915	/* I/O operation lookup arrays. */
7916	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7917	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7918
7919	const uint32_t cbSize = s_aIOSize[uIOWidth];
7920	const uint32_t cbInstr = pVmxTransient->cbInstr;
7921	PVM pVM = pVCpu->CTX_SUFF(pVM);
7922	if (fIOString)
7923	{
7924	/* INS/OUTS - I/O String instruction. */
7925	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7926	/** @todo for now manually disassemble later optimize by getting the fields from
7927	* the VMCS. */
7928	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7929	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7930	* segment prefix info. */
7931	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7932	if (RT_SUCCESS(rc))
7933	{
7934	if (fIOWrite)
7935	{
7936	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7937	(DISCPUMODE)pDis->uAddrMode, cbSize);
7938	rc = VBOXSTRICTRC_VAL(rc2);
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7940	}
7941	else
7942	{
7943	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7944	(DISCPUMODE)pDis->uAddrMode, cbSize);
7945	rc = VBOXSTRICTRC_VAL(rc2);
7946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7947	}
7948	}
7949	else
7950	{
7951	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7952	rc = VINF_EM_RAW_EMULATE_INSTR;
7953	}
7954	}
7955	else
7956	{
7957	/* IN/OUT - I/O instruction. */
7958	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7959	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7960	if (fIOWrite)
7961	{
7962	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7963	rc = VBOXSTRICTRC_VAL(rc2);
7964	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7965	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7966	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7967	}
7968	else
7969	{
7970	uint32_t u32Result = 0;
7971	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7972	rc = VBOXSTRICTRC_VAL(rc2);
7973	if (IOM_SUCCESS(rc))
7974	{
7975	/* Save result of I/O IN instr. in AL/AX/EAX. */
7976	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7977	}
7978	else if (rc == VINF_IOM_R3_IOPORT_READ)
7979	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7980	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7981	}
7982	}
7983
7984	if (IOM_SUCCESS(rc))
7985	{
7986	pMixedCtx->rip += cbInstr;
7987	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7988	if (RT_LIKELY(rc == VINF_SUCCESS))
7989	{
7990	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7991	AssertRCReturn(rc, rc);
7992
7993	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7994	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7995	{
7996	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7997	for (unsigned i = 0; i < 4; i++)
7998	{
7999	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8000	if ( ( uIOPort >= pMixedCtx->dr[i]
8001	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8002	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8003	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8004	{
8005	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8006	uint64_t uDR6 = ASMGetDR6();
8007
8008	/* Clear all breakpoint status flags and set the one we just hit. */
8009	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8010	uDR6 \|= (uint64_t)RT_BIT(i);
8011
8012	/*
8013	* Note: AMD64 Architecture Programmer's Manual 13.1:
8014	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8015	* be cleared by software after the contents have been read.
8016	*/
8017	ASMSetDR6(uDR6);
8018
8019	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8020	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8021
8022	/* Paranoia. */
8023	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8024	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8025	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8026
8027	/* Resync DR7 */
8028	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8029	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8030
8031	/* Set #DB to be injected into the VM and continue guest execution. */
8032	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8033	break;
8034	}
8035	}
8036	}
8037	}
8038	}
8039
8040	#ifdef DEBUG
8041	if (rc == VINF_IOM_R3_IOPORT_READ)
8042	Assert(!fIOWrite);
8043	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8044	Assert(fIOWrite);
8045	else
8046	{
8047	AssertMsg( RT_FAILURE(rc)
8048	\|\| rc == VINF_SUCCESS
8049	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8050	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8051	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8052	}
8053	#endif
8054
8055	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8056	return rc;
8057	}
8058
8059
8060	/**
8061	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8062	* VM-exit.
8063	*/
8064	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8065	{
8066	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8067
8068	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8069	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8070	AssertRCReturn(rc, rc);
8071	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8072	{
8073	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8074	AssertRCReturn(rc, rc);
8075	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8076	{
8077	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8078	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8079	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8080	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8081	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8082	{
8083	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8084	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8085
8086	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8087	Assert(!pVCpu->hm.s.Event.fPending);
8088	pVCpu->hm.s.Event.fPending = true;
8089	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8090	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8091	AssertRCReturn(rc, rc);
8092	if (fErrorCodeValid)
8093	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8094	else
8095	pVCpu->hm.s.Event.u32ErrCode = 0;
8096	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8097	&& uVector == X86_XCPT_PF)
8098	{
8099	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8100	}
8101	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8102	}
8103	}
8104	}
8105	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8106	* emulation. */
8107	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8108	return VERR_EM_INTERPRETER;
8109	}
8110
8111
8112	/**
8113	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8114	*/
8115	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8116	{
8117	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8118	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8119	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8120	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8121	AssertRCReturn(rc, rc);
8122	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8123	return VINF_EM_DBG_STOP;
8124	}
8125
8126
8127	/**
8128	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8129	*/
8130	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8131	{
8132	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8133
8134	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8135	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8136	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8137	return VINF_SUCCESS;
8138	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8139	return rc;
8140
8141	#if 0
8142	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8143	* just sync the whole thing. */
8144	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8145	#else
8146	/* Aggressive state sync. for now. */
8147	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8148	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8149	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8150	#endif
8151	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8152	AssertRCReturn(rc, rc);
8153
8154	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8155	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8156	switch (uAccessType)
8157	{
8158	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8159	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8160	{
8161	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8162	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8163	{
8164	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8165	}
8166
8167	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8168	GCPhys &= PAGE_BASE_GC_MASK;
8169	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8170	PVM pVM = pVCpu->CTX_SUFF(pVM);
8171	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8172	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8173
8174	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8175	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8176	CPUMCTX2CORE(pMixedCtx), GCPhys);
8177	rc = VBOXSTRICTRC_VAL(rc2);
8178	Log(("ApicAccess rc=%d\n", rc));
8179	if ( rc == VINF_SUCCESS
8180	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8181	\|\| rc == VERR_PAGE_NOT_PRESENT)
8182	{
8183	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8184	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8185	rc = VINF_SUCCESS;
8186	}
8187	break;
8188	}
8189
8190	default:
8191	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8192	rc = VINF_EM_RAW_EMULATE_INSTR;
8193	break;
8194	}
8195
8196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8197	return rc;
8198	}
8199
8200
8201	/**
8202	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8203	* VM-exit.
8204	*/
8205	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8206	{
8207	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8208
8209	/* We should -not- get this VM-exit if the guest is debugging. */
8210	if (CPUMIsGuestDebugStateActive(pVCpu))
8211	{
8212	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8213	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8214	}
8215
8216	int rc = VERR_INTERNAL_ERROR_5;
8217	if ( !DBGFIsStepping(pVCpu)
8218	&& !CPUMIsHyperDebugStateActive(pVCpu))
8219	{
8220	/* Don't intercept MOV DRx. */
8221	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8223	AssertRCReturn(rc, rc);
8224
8225	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8226	PVM pVM = pVCpu->CTX_SUFF(pVM);
8227	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8228	AssertRC(rc);
8229	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8230
8231	#ifdef VBOX_WITH_STATISTICS
8232	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8233	AssertRCReturn(rc, rc);
8234	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8236	else
8237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8238	#endif
8239	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8240	return VINF_SUCCESS;
8241	}
8242
8243	/*
8244	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8245	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8246	*/
8247	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8248	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8249	AssertRCReturn(rc, rc);
8250
8251	PVM pVM = pVCpu->CTX_SUFF(pVM);
8252	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8253	{
8254	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8255	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8256	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8257	if (RT_SUCCESS(rc))
8258	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8260	}
8261	else
8262	{
8263	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8264	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8265	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8267	}
8268
8269	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8270	if (RT_SUCCESS(rc))
8271	{
8272	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8273	AssertRCReturn(rc2, rc2);
8274	}
8275	return rc;
8276	}
8277
8278
8279	/**
8280	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8281	* Conditional VM-exit.
8282	*/
8283	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8284	{
8285	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8286	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8287
8288	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8289	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8290	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8291	return VINF_SUCCESS;
8292	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8293	return rc;
8294
8295	RTGCPHYS GCPhys = 0;
8296	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8297
8298	#if 0
8299	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8300	#else
8301	/* Aggressive state sync. for now. */
8302	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8303	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8304	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8305	#endif
8306	AssertRCReturn(rc, rc);
8307
8308	/*
8309	* If we succeed, resume guest execution.
8310	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8311	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8312	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8313	* weird case. See @bugref{6043}.
8314	*/
8315	PVM pVM = pVCpu->CTX_SUFF(pVM);
8316	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8317	rc = VBOXSTRICTRC_VAL(rc2);
8318	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8319	if ( rc == VINF_SUCCESS
8320	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8321	\|\| rc == VERR_PAGE_NOT_PRESENT)
8322	{
8323	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8324	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8325	return VINF_SUCCESS;
8326	}
8327	return rc;
8328	}
8329
8330
8331	/**
8332	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8333	* VM-exit.
8334	*/
8335	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8336	{
8337	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8338	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8339
8340	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8341	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8342	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8343	return VINF_SUCCESS;
8344	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8345	return rc;
8346
8347	RTGCPHYS GCPhys = 0;
8348	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8349	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8350	#if 0
8351	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8352	#else
8353	/* Aggressive state sync. for now. */
8354	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8355	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8356	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8357	#endif
8358	AssertRCReturn(rc, rc);
8359
8360	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8361	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8362
8363	RTGCUINT uErrorCode = 0;
8364	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8365	uErrorCode \|= X86_TRAP_PF_ID;
8366	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8367	uErrorCode \|= X86_TRAP_PF_RW;
8368	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8369	uErrorCode \|= X86_TRAP_PF_P;
8370
8371	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8372
8373	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8374	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8375
8376	/* Handle the pagefault trap for the nested shadow table. */
8377	PVM pVM = pVCpu->CTX_SUFF(pVM);
8378	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8379	TRPMResetTrap(pVCpu);
8380
8381	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8382	if ( rc == VINF_SUCCESS
8383	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8384	\|\| rc == VERR_PAGE_NOT_PRESENT)
8385	{
8386	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8388	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8389	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8390	return VINF_SUCCESS;
8391	}
8392
8393	Log(("EPT return to ring-3 rc=%d\n"));
8394	return rc;
8395	}
8396
8397
8398	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8399	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8400	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8401	/**
8402	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8403	*/
8404	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8405	{
8406	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8407	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8408
8409	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8410	AssertRCReturn(rc, rc);
8411
8412	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8413	{
8414	/* Old-style FPU error reporting needs some extra work. */
8415	/** @todo don't fall back to the recompiler, but do it manually. */
8416	return VERR_EM_INTERPRETER;
8417	}
8418	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8419	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8420	return rc;
8421	}
8422
8423
8424	/**
8425	* VM-exit exception handler for #BP (Breakpoint exception).
8426	*/
8427	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8428	{
8429	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8430	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8431
8432	/** @todo Try optimize this by not saving the entire guest state unless
8433	* really needed. */
8434	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8435	AssertRCReturn(rc, rc);
8436
8437	PVM pVM = pVCpu->CTX_SUFF(pVM);
8438	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8439	if (rc == VINF_EM_RAW_GUEST_TRAP)
8440	{
8441	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8442	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8443	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8444	AssertRCReturn(rc, rc);
8445
8446	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8447	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8448	}
8449
8450	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8451	return rc;
8452	}
8453
8454
8455	/**
8456	* VM-exit exception handler for #DB (Debug exception).
8457	*/
8458	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8459	{
8460	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8462
8463	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8464	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8465	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8466	AssertRCReturn(rc, rc);
8467
8468	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8469	uint64_t uDR6 = X86_DR6_INIT_VAL;
8470	uDR6 \|= (pVmxTransient->uExitQualification
8471	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8472	PVM pVM = pVCpu->CTX_SUFF(pVM);
8473	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8474	if (rc == VINF_EM_RAW_GUEST_TRAP)
8475	{
8476	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8477	pMixedCtx->dr[6] = uDR6;
8478
8479	if (CPUMIsGuestDebugStateActive(pVCpu))
8480	ASMSetDR6(pMixedCtx->dr[6]);
8481
8482	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8483
8484	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8485	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8486
8487	/* Paranoia. */
8488	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8489	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8490	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8491
8492	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8493	AssertRCReturn(rc,rc);
8494
8495	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8496	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8497	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8498	AssertRCReturn(rc2, rc2);
8499	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8500	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8501	rc = VINF_SUCCESS;
8502	}
8503
8504	return rc;
8505	}
8506
8507
8508	/**
8509	* VM-exit exception handler for #NM (Device-not-available exception: floating
8510	* point exception).
8511	*/
8512	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8513	{
8514	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8515
8516	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8517	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8518	#endif
8519
8520	/* We require CR0 and EFER. EFER is always up-to-date. */
8521	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8522	AssertRCReturn(rc, rc);
8523
8524	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8525	PVM pVM = pVCpu->CTX_SUFF(pVM);
8526	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8527	if (rc == VINF_SUCCESS)
8528	{
8529	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8531	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8532	return VINF_SUCCESS;
8533	}
8534
8535	/* Forward #NM to the guest. */
8536	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8537	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8538	AssertRCReturn(rc, rc);
8539	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8540	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8541	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8542	return rc;
8543	}
8544
8545
8546	/**
8547	* VM-exit exception handler for #GP (General-protection exception).
8548	*
8549	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8550	*/
8551	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8552	{
8553	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8554	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8555
8556	int rc = VERR_INTERNAL_ERROR_5;
8557	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8558	{
8559	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8560	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8561	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8562	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8563	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8564	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8565	AssertRCReturn(rc, rc);
8566	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8567	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8568	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8569	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8570	return rc;
8571	#else
8572	/* We don't intercept #GP. */
8573	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8574	return VERR_VMX_UNEXPECTED_EXCEPTION;
8575	#endif
8576	}
8577
8578	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8579	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8580
8581	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8582	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8583	AssertRCReturn(rc, rc);
8584
8585	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8586	uint32_t cbOp = 0;
8587	PVM pVM = pVCpu->CTX_SUFF(pVM);
8588	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8589	if (RT_SUCCESS(rc))
8590	{
8591	rc = VINF_SUCCESS;
8592	Assert(cbOp == pDis->cbInstr);
8593	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8594	switch (pDis->pCurInstr->uOpcode)
8595	{
8596	case OP_CLI:
8597	pMixedCtx->eflags.Bits.u1IF = 0;
8598	pMixedCtx->rip += pDis->cbInstr;
8599	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8601	break;
8602
8603	case OP_STI:
8604	pMixedCtx->eflags.Bits.u1IF = 1;
8605	pMixedCtx->rip += pDis->cbInstr;
8606	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8607	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8608	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8609	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8610	break;
8611
8612	case OP_HLT:
8613	rc = VINF_EM_HALT;
8614	pMixedCtx->rip += pDis->cbInstr;
8615	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8617	break;
8618
8619	case OP_POPF:
8620	{
8621	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8622	uint32_t cbParm = 0;
8623	uint32_t uMask = 0;
8624	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8625	{
8626	cbParm = 4;
8627	uMask = 0xffffffff;
8628	}
8629	else
8630	{
8631	cbParm = 2;
8632	uMask = 0xffff;
8633	}
8634
8635	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8636	RTGCPTR GCPtrStack = 0;
8637	X86EFLAGS uEflags;
8638	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8639	&GCPtrStack);
8640	if (RT_SUCCESS(rc))
8641	{
8642	Assert(sizeof(uEflags.u32) >= cbParm);
8643	uEflags.u32 = 0;
8644	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8645	}
8646	if (RT_FAILURE(rc))
8647	{
8648	rc = VERR_EM_INTERPRETER;
8649	break;
8650	}
8651	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8652	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8653	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8654	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8655	pMixedCtx->eflags.Bits.u1RF = 0;
8656	pMixedCtx->esp += cbParm;
8657	pMixedCtx->esp &= uMask;
8658	pMixedCtx->rip += pDis->cbInstr;
8659	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8660	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8661	break;
8662	}
8663
8664	case OP_PUSHF:
8665	{
8666	uint32_t cbParm = 0;
8667	uint32_t uMask = 0;
8668	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8669	{
8670	cbParm = 4;
8671	uMask = 0xffffffff;
8672	}
8673	else
8674	{
8675	cbParm = 2;
8676	uMask = 0xffff;
8677	}
8678
8679	/* Get the stack pointer & push the contents of eflags onto the stack. */
8680	RTGCPTR GCPtrStack = 0;
8681	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8682	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8683	if (RT_FAILURE(rc))
8684	{
8685	rc = VERR_EM_INTERPRETER;
8686	break;
8687	}
8688	X86EFLAGS uEflags;
8689	uEflags = pMixedCtx->eflags;
8690	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8691	uEflags.Bits.u1RF = 0;
8692	uEflags.Bits.u1VM = 0;
8693
8694	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8695	if (RT_FAILURE(rc))
8696	{
8697	rc = VERR_EM_INTERPRETER;
8698	break;
8699	}
8700	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8701	pMixedCtx->esp -= cbParm;
8702	pMixedCtx->esp &= uMask;
8703	pMixedCtx->rip += pDis->cbInstr;
8704	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8705	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8706	break;
8707	}
8708
8709	case OP_IRET:
8710	{
8711	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8712	* instruction reference. */
8713	RTGCPTR GCPtrStack = 0;
8714	uint32_t uMask = 0xffff;
8715	uint16_t aIretFrame[3];
8716	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8717	{
8718	rc = VERR_EM_INTERPRETER;
8719	break;
8720	}
8721	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8722	&GCPtrStack);
8723	if (RT_SUCCESS(rc))
8724	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8725	if (RT_FAILURE(rc))
8726	{
8727	rc = VERR_EM_INTERPRETER;
8728	break;
8729	}
8730	pMixedCtx->eip = 0;
8731	pMixedCtx->ip = aIretFrame[0];
8732	pMixedCtx->cs.Sel = aIretFrame[1];
8733	pMixedCtx->cs.ValidSel = aIretFrame[1];
8734	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8735	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8736	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8737	pMixedCtx->sp += sizeof(aIretFrame);
8738	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8739	\| HM_CHANGED_GUEST_RFLAGS;
8740	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8741	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8742	break;
8743	}
8744
8745	case OP_INT:
8746	{
8747	uint16_t uVector = pDis->Param1.uValue & 0xff;
8748	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8750	break;
8751	}
8752
8753	case OP_INTO:
8754	{
8755	if (pMixedCtx->eflags.Bits.u1OF)
8756	{
8757	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8758	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8759	}
8760	break;
8761	}
8762
8763	default:
8764	{
8765	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8766	EMCODETYPE_SUPERVISOR);
8767	rc = VBOXSTRICTRC_VAL(rc2);
8768	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8769	Log(("#GP rc=%Rrc\n", rc));
8770	break;
8771	}
8772	}
8773	}
8774	else
8775	rc = VERR_EM_INTERPRETER;
8776
8777	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8778	("#GP Unexpected rc=%Rrc\n", rc));
8779	return rc;
8780	}
8781
8782
8783	/**
8784	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8785	* the exception reported in the VMX transient structure back into the VM.
8786	*
8787	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8788	* VMX transient structure to be up-to-date.
8789	*/
8790	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8791	{
8792	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8793
8794	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8795	hmR0VmxCheckExitDueToEventDelivery(). */
8796	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8797	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8798	AssertRCReturn(rc, rc);
8799	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8800	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8801	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8802	return VINF_SUCCESS;
8803	}
8804
8805
8806	/**
8807	* VM-exit exception handler for #PF (Page-fault exception).
8808	*/
8809	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8810	{
8811	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8812	PVM pVM = pVCpu->CTX_SUFF(pVM);
8813	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8814	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8815	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8816	AssertRCReturn(rc, rc);
8817
8818	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8819	if (pVM->hm.s.fNestedPaging)
8820	{
8821	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8822	{
8823	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8824	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8825	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8826	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8827	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8828	}
8829	else
8830	{
8831	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8832	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8833	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8834	Log(("Pending #DF due to vectoring #PF. NP\n"));
8835	}
8836	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8837	return rc;
8838	}
8839	#else
8840	Assert(!pVM->hm.s.fNestedPaging);
8841	#endif
8842
8843	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8844	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8845	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8846	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8847	AssertRCReturn(rc, rc);
8848	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8849	if ( pVM->hm.s.fTRPPatchingAllowed
8850	&& pVM->hm.s.pGuestPatchMem
8851	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8852	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8853	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8854	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8855	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8856	{
8857	RTGCPHYS GCPhys;
8858	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8859	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8860	if ( rc == VINF_SUCCESS
8861	&& GCPhys == GCPhysApicBase)
8862	{
8863	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8864	AssertRCReturn(rc, rc);
8865
8866	/* Only attempt to patch the instruction once. */
8867	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8868	if (!pPatch)
8869	return VINF_EM_HM_PATCH_TPR_INSTR;
8870	}
8871	}
8872	#endif
8873
8874	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8875	AssertRCReturn(rc, rc);
8876
8877	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8878	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8879
8880	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8881	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8882	(RTGCPTR)pVmxTransient->uExitQualification);
8883
8884	Log(("#PF: rc=%Rrc\n", rc));
8885	if (rc == VINF_SUCCESS)
8886	{
8887	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8888	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8889	* memory? We don't update the whole state here... */
8890	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8891	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8892	TRPMResetTrap(pVCpu);
8893	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8894	return rc;
8895	}
8896	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8897	{
8898	if (!pVmxTransient->fVectoringPF)
8899	{
8900	/* It's a guest page fault and needs to be reflected to the guest. */
8901	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8902	TRPMResetTrap(pVCpu);
8903	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8904	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8905	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8906	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8907	}
8908	else
8909	{
8910	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8911	TRPMResetTrap(pVCpu);
8912	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8913	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8914	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8915	}
8916
8917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8918	return VINF_SUCCESS;
8919	}
8920
8921	TRPMResetTrap(pVCpu);
8922	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8923	return rc;
8924	}
8925

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45852

Download in other formats: