HMVMXR0.cpp@ 45676

Last change on this file since 45676 was 45668, checked in by vboxsync, 12 years ago
VMMR0/HMVMXR0: Don't intercept CR0.WP when we have nested paging.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 376.5 KB

Line
1	/* $Id: HMVMXR0.cpp 45668 2013-04-22 15:15:42Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_TRAP_ALL_EXCEPTIONS
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define HMVMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define HMVMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/** Use the function table. */
56	#define HMVMX_USE_FUNCTION_TABLE
57
58	/** This bit indicates the segment selector is unusable in VT-x. */
59	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
60
61	/** Determine which tagged-TLB flush handler to use. */
62	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67	/**
68	* Updated-guest-state flags.
69	*/
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DEBUG \
102	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
103	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110
111	/**
112	* Flags to skip redundant reads of some common VMCS fields that are not part of
113	* the guest-CPU state but are in the transient structure.
114	*/
115	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121
122	/**
123	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
124	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
125	* we have Nested Paging support.
126	*/
127	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
128	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
129	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
130	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
131	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
133	\| RT_BIT(X86_XCPT_XF))
134
135	/**
136	* Exception bitmap mask for all contributory exceptions.
137	*/
138	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
139	\| RT_BIT(X86_XCPT_DE))
140
141	/** Maximum VM-instruction error number. */
142	#define HMVMX_INSTR_ERROR_MAX 28
143
144
145	/*******************************************************************************
146	* Structures and Typedefs *
147	*******************************************************************************/
148	/**
149	* A state structure for holding miscellaneous information across
150	* VMX non-root operation and restored after the transition.
151	*/
152	typedef struct VMXTRANSIENT
153	{
154	/** The host's rflags/eflags. */
155	RTCCUINTREG uEFlags;
156	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
157	uint64_t u64LStarMsr;
158	/** The guest's TPR value used for TPR shadowing. */
159	uint8_t u8GuestTpr;
160
161	/** The basic VM-exit reason. */
162	uint16_t uExitReason;
163	/** The VM-exit exit qualification. */
164	RTGCUINTPTR uExitQualification;
165	/** The VM-exit interruption error code. */
166	uint32_t uExitIntrErrorCode;
167
168	/** The VM-exit interruption-information field. */
169	uint32_t uExitIntrInfo;
170	/** Whether the VM-entry failed or not. */
171	bool fVMEntryFailed;
172	/** The VM-exit instruction-length field. */
173	uint32_t cbInstr;
174
175	/** The VM-entry interruption-information field. */
176	uint32_t uEntryIntrInfo;
177	/** The VM-entry exception error code field. */
178	uint32_t uEntryXcptErrorCode;
179	/** The VM-entry instruction length field. */
180	uint32_t cbEntryInstr;
181
182	/** IDT-vectoring information field. */
183	uint32_t uIdtVectoringInfo;
184	/** IDT-vectoring error code. */
185	uint32_t uIdtVectoringErrorCode;
186
187	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
188	uint32_t fVmcsFieldsRead;
189	/** Whether TSC-offsetting should be setup before VM-entry. */
190	bool fUpdateTscOffsettingAndPreemptTimer;
191	/** Whether the VM-exit was caused by a page-fault during delivery of a
192	* contributary exception or a page-fault. */
193	bool fVectoringPF;
194	} VMXTRANSIENT, *PVMXTRANSIENT;
195
196	/**
197	* MSR-bitmap read permissions.
198	*/
199	typedef enum VMXMSREXITREAD
200	{
201	/** Reading this MSR causes a VM-exit. */
202	VMXMSREXIT_INTERCEPT_READ = 0xb,
203	/** Reading this MSR does not cause a VM-exit. */
204	VMXMSREXIT_PASSTHRU_READ
205	} VMXMSREXITREAD;
206
207	/**
208	* MSR-bitmap write permissions.
209	*/
210	typedef enum VMXMSREXITWRITE
211	{
212	/** Writing to this MSR causes a VM-exit. */
213	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
214	/** Writing to this MSR does not cause a VM-exit. */
215	VMXMSREXIT_PASSTHRU_WRITE
216	} VMXMSREXITWRITE;
217
218
219	/*******************************************************************************
220	* Internal Functions *
221	*******************************************************************************/
222	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
223	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
224	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
225	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
226	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
227	#endif
228	#ifndef HMVMX_USE_FUNCTION_TABLE
229	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
230	#endif
231
232	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276
277	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284
285
286	/*******************************************************************************
287	* Global Variables *
288	*******************************************************************************/
289	#ifdef HMVMX_USE_FUNCTION_TABLE
290	/**
291	* VM-exit handler.
292	*
293	* @returns VBox status code.
294	* @param pVCpu Pointer to the VMCPU.
295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
296	* out-of-sync. Make sure to update the required
297	* fields before using them.
298	* @param pVmxTransient Pointer to the VMX-transient structure.
299	*/
300	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	/** Pointer to VM-exit handler. */
302	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
303
304	/**
305	* VMX_EXIT dispatch table.
306	*/
307	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
308	{
309	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
310	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
311	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
312	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
313	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
314	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
315	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
316	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
317	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
318	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
319	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
320	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
321	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
322	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
323	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
324	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
325	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
326	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
327	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
328	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
329	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
330	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
331	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
332	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
333	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
334	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
335	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
336	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
337	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
338	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
339	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
340	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
341	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
342	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
343	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
344	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
345	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
346	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
347	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
348	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
349	/* 40 UNDEFINED */ hmR0VmxExitPause,
350	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
351	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
352	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
353	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
354	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
355	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
356	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
357	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
358	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
359	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
360	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
361	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
362	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
363	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
364	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
365	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
366	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
367	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
368	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
369	};
370	#endif /* HMVMX_USE_FUNCTION_TABLE */
371
372	#ifdef VBOX_STRICT
373	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
374	{
375	/* 0 */ "(Not Used)",
376	/* 1 */ "VMCALL executed in VMX root operation.",
377	/* 2 */ "VMCLEAR with invalid physical address.",
378	/* 3 */ "VMCLEAR with VMXON pointer.",
379	/* 4 */ "VMLAUNCH with non-clear VMCS.",
380	/* 5 */ "VMRESUME with non-launched VMCS.",
381	/* 6 */ "VMRESUME after VMXOFF",
382	/* 7 */ "VM entry with invalid control fields.",
383	/* 8 */ "VM entry with invalid host state fields.",
384	/* 9 */ "VMPTRLD with invalid physical address.",
385	/* 10 */ "VMPTRLD with VMXON pointer.",
386	/* 11 */ "VMPTRLD with incorrect revision identifier.",
387	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
388	/* 13 */ "VMWRITE to read-only VMCS component.",
389	/* 14 */ "(Not Used)",
390	/* 15 */ "VMXON executed in VMX root operation.",
391	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
392	/* 17 */ "VM entry with non-launched executing VMCS.",
393	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
394	/* 19 */ "VMCALL with non-clear VMCS.",
395	/* 20 */ "VMCALL with invalid VM-exit control fields.",
396	/* 21 */ "(Not Used)",
397	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
398	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
399	/* 24 */ "VMCALL with invalid SMM-monitor features.",
400	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
401	/* 26 */ "VM entry with events blocked by MOV SS.",
402	/* 27 */ "(Not Used)",
403	/* 28 */ "Invalid operand to INVEPT/INVVPID."
404	};
405	#endif /* VBOX_STRICT */
406
407
408
409	/**
410	* Updates the VM's last error record. If there was a VMX instruction error,
411	* reads the error data from the VMCS and updates VCPU's last error record as
412	* well.
413	*
414	* @param pVM Pointer to the VM.
415	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
416	* VERR_VMX_UNABLE_TO_START_VM or
417	* VERR_VMX_INVALID_VMCS_FIELD).
418	* @param rc The error code.
419	*/
420	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
421	{
422	AssertPtr(pVM);
423	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
424	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
425	{
426	AssertPtrReturnVoid(pVCpu);
427	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
428	}
429	pVM->hm.s.lLastError = rc;
430	}
431
432
433	/**
434	* Reads the VM-entry interruption-information field from the VMCS into the VMX
435	* transient structure.
436	*
437	* @returns VBox status code.
438	* @param pVmxTransient Pointer to the VMX transient structure.
439	*
440	* @remarks No-long-jump zone!!!
441	*/
442	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
443	{
444	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
445	AssertRCReturn(rc, rc);
446	return VINF_SUCCESS;
447	}
448
449
450	/**
451	* Reads the VM-entry exception error code field from the VMCS into
452	* the VMX transient structure.
453	*
454	* @returns VBox status code.
455	* @param pVmxTransient Pointer to the VMX transient structure.
456	*
457	* @remarks No-long-jump zone!!!
458	*/
459	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
460	{
461	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
462	AssertRCReturn(rc, rc);
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Reads the VM-entry exception error code field from the VMCS into
469	* the VMX transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVCpu Pointer to the VMCPU.
473	* @param pVmxTransient Pointer to the VMX transient structure.
474	*
475	* @remarks No-long-jump zone!!!
476	*/
477	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
478	{
479	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
480	AssertRCReturn(rc, rc);
481	return VINF_SUCCESS;
482	}
483
484
485	/**
486	* Reads the VM-exit interruption-information field from the VMCS into the VMX
487	* transient structure.
488	*
489	* @returns VBox status code.
490	* @param pVCpu Pointer to the VMCPU.
491	* @param pVmxTransient Pointer to the VMX transient structure.
492	*/
493	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
494	{
495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
496	{
497	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
498	AssertRCReturn(rc, rc);
499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
500	}
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption error code from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit instruction length field from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the exit qualification from the VMCS into the VMX transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVCpu Pointer to the VMCPU.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*/
552	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
553	{
554	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
555	{
556	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
557	AssertRCReturn(rc, rc);
558	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
559	}
560	return VINF_SUCCESS;
561	}
562
563
564	/**
565	* Reads the IDT-vectoring information field from the VMCS into the VMX
566	* transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVmxTransient Pointer to the VMX transient structure.
570	*
571	* @remarks No-long-jump zone!!!
572	*/
573	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
574	{
575	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
576	{
577	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
578	AssertRCReturn(rc, rc);
579	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
580	}
581	return VINF_SUCCESS;
582	}
583
584
585	/**
586	* Reads the IDT-vectoring error code from the VMCS into the VMX
587	* transient structure.
588	*
589	* @returns VBox status code.
590	* @param pVmxTransient Pointer to the VMX transient structure.
591	*/
592	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
593	{
594	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
595	{
596	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
597	AssertRCReturn(rc, rc);
598	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
599	}
600	return VINF_SUCCESS;
601	}
602
603
604	/**
605	* Enters VMX root mode operation on the current CPU.
606	*
607	* @returns VBox status code.
608	* @param pVM Pointer to the VM (optional, can be NULL, after
609	* a resume).
610	* @param HCPhysCpuPage Physical address of the VMXON region.
611	* @param pvCpuPage Pointer to the VMXON region.
612	*/
613	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
614	{
615	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
616	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
617
618	if (pVM)
619	{
620	/* Write the VMCS revision dword to the VMXON region. */
621	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
622	}
623
624	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
625	RTCCUINTREG fFlags = ASMIntDisableFlags();
626
627	/* Enable the VMX bit in CR4 if necessary. */
628	RTCCUINTREG uCr4 = ASMGetCR4();
629	if (!(uCr4 & X86_CR4_VMXE))
630	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
631
632	/* Enter VMX root mode. */
633	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
634	if (RT_FAILURE(rc))
635	ASMSetCR4(uCr4);
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Exits VMX root mode operation on the current CPU.
645	*
646	* @returns VBox status code.
647	*/
648	static int hmR0VmxLeaveRootMode(void)
649	{
650	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
651	RTCCUINTREG fFlags = ASMIntDisableFlags();
652	int rc = VINF_SUCCESS;
653
654	/* If we're for some reason not in VMX root mode, then don't leave it. */
655	if (ASMGetCR4() & X86_CR4_VMXE)
656	{
657	/* Exit VMX root mode and clear the VMX bit in CR4 */
658	VMXDisable();
659	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
660	}
661	else
662	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
663
664	/* Restore interrupts. */
665	ASMSetFlags(fFlags);
666	return rc;
667	}
668
669
670	/**
671	* Allocates and maps one physically contiguous page. The allocated page is
672	* zero'd out. (Used by various VT-x structures).
673	*
674	* @returns IPRT status code.
675	* @param pMemObj Pointer to the ring-0 memory object.
676	* @param ppVirt Where to store the virtual address of the
677	* allocation.
678	* @param pPhys Where to store the physical address of the
679	* allocation.
680	*/
681	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
682	{
683	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
684	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
685	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
686
687	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
688	if (RT_FAILURE(rc))
689	return rc;
690	ppVirt = RTR0MemObjAddress(pMemObj);
691	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
692	ASMMemZero32(*ppVirt, PAGE_SIZE);
693	return VINF_SUCCESS;
694	}
695
696
697	/**
698	* Frees and unmaps an allocated physical page.
699	*
700	* @param pMemObj Pointer to the ring-0 memory object.
701	* @param ppVirt Where to re-initialize the virtual address of
702	* allocation as 0.
703	* @param pHCPhys Where to re-initialize the physical address of the
704	* allocation as 0.
705	*/
706	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
707	{
708	AssertPtr(pMemObj);
709	AssertPtr(ppVirt);
710	AssertPtr(pHCPhys);
711	if (*pMemObj != NIL_RTR0MEMOBJ)
712	{
713	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
714	AssertRC(rc);
715	*pMemObj = NIL_RTR0MEMOBJ;
716	*ppVirt = 0;
717	*pHCPhys = 0;
718	}
719	}
720
721
722	/**
723	* Worker function to free VT-x related structures.
724	*
725	* @returns IPRT status code.
726	* @param pVM Pointer to the VM.
727	*/
728	static void hmR0VmxStructsFree(PVM pVM)
729	{
730	for (VMCPUID i = 0; i < pVM->cCpus; i++)
731	{
732	PVMCPU pVCpu = &pVM->aCpus[i];
733	AssertPtr(pVCpu);
734
735	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
736	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
737
738	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
739	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
740
741	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
742	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
743	}
744
745	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
748	#endif
749	}
750
751
752	/**
753	* Worker function to allocate VT-x related VM structures.
754	*
755	* @returns IPRT status code.
756	* @param pVM Pointer to the VM.
757	*/
758	static int hmR0VmxStructsAlloc(PVM pVM)
759	{
760	/*
761	* Initialize members up-front so we can cleanup properly on allocation failure.
762	*/
763	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
764	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
765	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
766	pVM->hm.s.vmx.HCPhys##a_Name = 0;
767
768	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
769	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
770	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
771	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
772
773	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
774	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
775	#endif
776	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
777
778	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
779	for (VMCPUID i = 0; i < pVM->cCpus; i++)
780	{
781	PVMCPU pVCpu = &pVM->aCpus[i];
782	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
783	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
784	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
785	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
786	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
787	}
788	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
789	#undef VMXLOCAL_INIT_VM_MEMOBJ
790
791	/*
792	* Allocate all the VT-x structures.
793	*/
794	int rc = VINF_SUCCESS;
795	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
796	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
797	if (RT_FAILURE(rc))
798	goto cleanup;
799	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
800	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
801	#endif
802
803	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
804	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
805	{
806	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
807	&pVM->hm.s.vmx.HCPhysApicAccess);
808	if (RT_FAILURE(rc))
809	goto cleanup;
810	}
811
812	/*
813	* Initialize per-VCPU VT-x structures.
814	*/
815	for (VMCPUID i =0; i < pVM->cCpus; i++)
816	{
817	PVMCPU pVCpu = &pVM->aCpus[i];
818	AssertPtr(pVCpu);
819
820	/* Allocate the VM control structure (VMCS). */
821	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
822	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825
826	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
827	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
828	{
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
830	&pVCpu->hm.s.vmx.HCPhysVirtApic);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	}
834
835	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
836	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
837	{
838	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
839	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
840	if (RT_FAILURE(rc))
841	goto cleanup;
842	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
843	}
844
845	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
846	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
847	if (RT_FAILURE(rc))
848	goto cleanup;
849
850	/* Allocate the VM-exit MSR-load page for the host MSRs. */
851	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
852	if (RT_FAILURE(rc))
853	goto cleanup;
854	}
855
856	return VINF_SUCCESS;
857
858	cleanup:
859	hmR0VmxStructsFree(pVM);
860	return rc;
861	}
862
863
864	/**
865	* Does global VT-x initialization (called during module initialization).
866	*
867	* @returns VBox status code.
868	*/
869	VMMR0DECL(int) VMXR0GlobalInit(void)
870	{
871	#ifdef HMVMX_USE_FUNCTION_TABLE
872	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
873	# ifdef VBOX_STRICT
874	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
875	Assert(g_apfnVMExitHandlers[i]);
876	# endif
877	#endif
878	return VINF_SUCCESS;
879	}
880
881
882	/**
883	* Does global VT-x termination (called during module termination).
884	*/
885	VMMR0DECL(void) VMXR0GlobalTerm()
886	{
887	/* Nothing to do currently. */
888	}
889
890
891	/**
892	* Sets up and activates VT-x on the current CPU.
893	*
894	* @returns VBox status code.
895	* @param pCpu Pointer to the global CPU info struct.
896	* @param pVM Pointer to the VM (can be NULL after a host resume
897	* operation).
898	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
899	* fEnabledByHost is true).
900	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
901	* @a fEnabledByHost is true).
902	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
903	* enable VT-x/AMD-V on the host.
904	*/
905	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
906	{
907	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
908	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
909
910	if (!fEnabledByHost)
911	{
912	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
913	if (RT_FAILURE(rc))
914	return rc;
915	}
916
917	/*
918	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
919	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
920	* each time while reusing a VPID after hitting the MaxASID limit once.
921	*/
922	if ( pVM
923	&& pVM->hm.s.vmx.fVpid
924	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
925	{
926	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
927	pCpu->fFlushAsidBeforeUse = false;
928	}
929	else
930	pCpu->fFlushAsidBeforeUse = true;
931
932	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
933	++pCpu->cTlbFlushes;
934
935	return VINF_SUCCESS;
936	}
937
938
939	/**
940	* Deactivates VT-x on the current CPU.
941	*
942	* @returns VBox status code.
943	* @param pCpu Pointer to the global CPU info struct.
944	* @param pvCpuPage Pointer to the VMXON region.
945	* @param HCPhysCpuPage Physical address of the VMXON region.
946	*/
947	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
948	{
949	NOREF(pCpu);
950	NOREF(pvCpuPage);
951	NOREF(HCPhysCpuPage);
952
953	hmR0VmxLeaveRootMode();
954	return VINF_SUCCESS;
955	}
956
957
958	/**
959	* Sets the permission bits for the specified MSR in the MSR bitmap.
960	*
961	* @param pVCpu Pointer to the VMCPU.
962	* @param uMSR The MSR value.
963	* @param enmRead Whether reading this MSR causes a VM-exit.
964	* @param enmWrite Whether writing this MSR causes a VM-exit.
965	*/
966	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
967	{
968	int32_t iBit;
969	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
970
971	/*
972	* Layout:
973	* 0x000 - 0x3ff - Low MSR read bits
974	* 0x400 - 0x7ff - High MSR read bits
975	* 0x800 - 0xbff - Low MSR write bits
976	* 0xc00 - 0xfff - High MSR write bits
977	*/
978	if (uMsr <= 0x00001FFF)
979	iBit = uMsr;
980	else if ( uMsr >= 0xC0000000
981	&& uMsr <= 0xC0001FFF)
982	{
983	iBit = (uMsr - 0xC0000000);
984	pbMsrBitmap += 0x400;
985	}
986	else
987	{
988	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
989	return;
990	}
991
992	Assert(iBit <= 0x1fff);
993	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
994	ASMBitSet(pbMsrBitmap, iBit);
995	else
996	ASMBitClear(pbMsrBitmap, iBit);
997
998	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
999	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1000	else
1001	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1002	}
1003
1004
1005	/**
1006	* Flushes the TLB using EPT.
1007	*
1008	* @returns VBox status code.
1009	* @param pVM Pointer to the VM.
1010	* @param pVCpu Pointer to the VMCPU.
1011	* @param enmFlush Type of flush.
1012	*/
1013	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1014	{
1015	AssertPtr(pVM);
1016	Assert(pVM->hm.s.fNestedPaging);
1017
1018	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1019
1020	uint64_t descriptor[2];
1021	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1022	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1023
1024	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1025	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1026	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1027	}
1028
1029
1030	/**
1031	* Flushes the TLB using VPID.
1032	*
1033	* @returns VBox status code.
1034	* @param pVM Pointer to the VM.
1035	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1036	* enmFlush).
1037	* @param enmFlush Type of flush.
1038	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1039	* on @a enmFlush).
1040	*/
1041	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1042	{
1043	AssertPtr(pVM);
1044	Assert(pVM->hm.s.vmx.fVpid);
1045
1046	uint64_t descriptor[2];
1047	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1048	{
1049	descriptor[0] = 0;
1050	descriptor[1] = 0;
1051	}
1052	else
1053	{
1054	AssertPtr(pVCpu);
1055	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1056	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1057	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1058	descriptor[1] = GCPtr;
1059	}
1060
1061	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1062	AssertMsg(rc == VINF_SUCCESS,
1063	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1064	if ( RT_SUCCESS(rc)
1065	&& pVCpu)
1066	{
1067	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1068	}
1069	}
1070
1071
1072	/**
1073	* Invalidates a guest page by guest virtual address. Only relevant for
1074	* EPT/VPID, otherwise there is nothing really to invalidate.
1075	*
1076	* @returns VBox status code.
1077	* @param pVM Pointer to the VM.
1078	* @param pVCpu Pointer to the VMCPU.
1079	* @param GCVirt Guest virtual address of the page to invalidate.
1080	*/
1081	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1082	{
1083	AssertPtr(pVM);
1084	AssertPtr(pVCpu);
1085	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1086
1087	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1088	if (!fFlushPending)
1089	{
1090	/*
1091	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1092	* See @bugref{6043} and @bugref{6177}.
1093	*
1094	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1095	* function maybe called in a loop with individual addresses.
1096	*/
1097	if (pVM->hm.s.vmx.fVpid)
1098	{
1099	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1100	{
1101	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1102	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1103	}
1104	else
1105	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1106	}
1107	else if (pVM->hm.s.fNestedPaging)
1108	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1109	}
1110
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1117	* otherwise there is nothing really to invalidate.
1118	*
1119	* @returns VBox status code.
1120	* @param pVM Pointer to the VM.
1121	* @param pVCpu Pointer to the VMCPU.
1122	* @param GCPhys Guest physical address of the page to invalidate.
1123	*/
1124	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1125	{
1126	LogFlowFunc(("%RGp\n", GCPhys));
1127
1128	/*
1129	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1130	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1131	* This function might be called in a loop.
1132	*/
1133	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1141	* case where neither EPT nor VPID is supported by the CPU.
1142	*
1143	* @param pVM Pointer to the VM.
1144	* @param pVCpu Pointer to the VMCPU.
1145	*
1146	* @remarks Called with interrupts disabled.
1147	*/
1148	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1149	{
1150	NOREF(pVM);
1151	AssertPtr(pVCpu);
1152	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1153	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1154
1155	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1156	AssertPtr(pCpu);
1157
1158	pVCpu->hm.s.TlbShootdown.cPages = 0;
1159	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1160	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1161	pVCpu->hm.s.fForceTLBFlush = false;
1162	return;
1163	}
1164
1165
1166	/**
1167	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	* @remarks All references to "ASID" in this function pertains to "VPID" in
1172	* Intel's nomenclature. The reason is, to avoid confusion in compare
1173	* statements since the host-CPU copies are named "ASID".
1174	*
1175	* @remarks Called with interrupts disabled.
1176	*/
1177	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1178	{
1179	AssertPtr(pVM);
1180	AssertPtr(pVCpu);
1181	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1182	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1183	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1184
1185	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1186	AssertPtr(pCpu);
1187
1188	/*
1189	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1190	* This can happen both for start & resume due to long jumps back to ring-3.
1191	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1192	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1193	*/
1194	bool fNewASID = false;
1195	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1196	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1197	{
1198	pVCpu->hm.s.fForceTLBFlush = true;
1199	fNewASID = true;
1200	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1201	}
1202
1203	/*
1204	* Check for explicit TLB shootdowns.
1205	*/
1206	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1207	{
1208	pVCpu->hm.s.fForceTLBFlush = true;
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1210	}
1211
1212	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1213	if (pVCpu->hm.s.fForceTLBFlush)
1214	{
1215	if (fNewASID)
1216	{
1217	++pCpu->uCurrentAsid;
1218	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1219	{
1220	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1221	pCpu->cTlbFlushes++;
1222	pCpu->fFlushAsidBeforeUse = true;
1223	}
1224
1225	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1226	if (pCpu->fFlushAsidBeforeUse)
1227	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1228	}
1229	else
1230	{
1231	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1232	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1233	else
1234	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1235	}
1236
1237	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1238	pVCpu->hm.s.fForceTLBFlush = false;
1239	}
1240	else
1241	{
1242	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1243	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1244	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1245	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1246
1247	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1248	* not be executed. See hmQueueInvlPage() where it is commented
1249	* out. Support individual entry flushing someday. */
1250	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1251	{
1252	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1253
1254	/*
1255	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1256	* as supported by the CPU.
1257	*/
1258	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1259	{
1260	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1261	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1262	}
1263	else
1264	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1265	}
1266	else
1267	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1268	}
1269	pVCpu->hm.s.TlbShootdown.cPages = 0;
1270	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1271
1272	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1273	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1274	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1275	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1276	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1277	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1278
1279	/* Update VMCS with the VPID. */
1280	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1281	AssertRC(rc);
1282	}
1283
1284
1285	/**
1286	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1287	*
1288	* @returns VBox status code.
1289	* @param pVM Pointer to the VM.
1290	* @param pVCpu Pointer to the VMCPU.
1291	*
1292	* @remarks Called with interrupts disabled.
1293	*/
1294	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1295	{
1296	AssertPtr(pVM);
1297	AssertPtr(pVCpu);
1298	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1299	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1300
1301	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1302	AssertPtr(pCpu);
1303
1304	/*
1305	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1306	* This can happen both for start & resume due to long jumps back to ring-3.
1307	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1308	*/
1309	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1310	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1311	{
1312	pVCpu->hm.s.fForceTLBFlush = true;
1313	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1314	}
1315
1316	/* Check for explicit TLB shootdown flushes. */
1317	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1318	{
1319	pVCpu->hm.s.fForceTLBFlush = true;
1320	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1321	}
1322
1323	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1324	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1325
1326	if (pVCpu->hm.s.fForceTLBFlush)
1327	{
1328	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1329	pVCpu->hm.s.fForceTLBFlush = false;
1330	}
1331	else
1332	{
1333	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1334	* not be executed. See hmQueueInvlPage() where it is commented
1335	* out. Support individual entry flushing someday. */
1336	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1337	{
1338	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1340	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1341	}
1342	else
1343	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1344	}
1345
1346	pVCpu->hm.s.TlbShootdown.cPages = 0;
1347	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1348	}
1349
1350
1351	/**
1352	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1353	*
1354	* @returns VBox status code.
1355	* @param pVM Pointer to the VM.
1356	* @param pVCpu Pointer to the VMCPU.
1357	*
1358	* @remarks Called with interrupts disabled.
1359	*/
1360	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1361	{
1362	AssertPtr(pVM);
1363	AssertPtr(pVCpu);
1364	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1365	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1366
1367	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1368
1369	/*
1370	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1371	* This can happen both for start & resume due to long jumps back to ring-3.
1372	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1373	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1374	*/
1375	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1376	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1377	{
1378	pVCpu->hm.s.fForceTLBFlush = true;
1379	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1380	}
1381
1382	/* Check for explicit TLB shootdown flushes. */
1383	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1384	{
1385	/*
1386	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1387	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1388	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1389	*/
1390	pVCpu->hm.s.fForceTLBFlush = true;
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1392	}
1393
1394	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1395	if (pVCpu->hm.s.fForceTLBFlush)
1396	{
1397	++pCpu->uCurrentAsid;
1398	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1399	{
1400	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1401	pCpu->fFlushAsidBeforeUse = true;
1402	pCpu->cTlbFlushes++;
1403	}
1404
1405	pVCpu->hm.s.fForceTLBFlush = false;
1406	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1407	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1408	if (pCpu->fFlushAsidBeforeUse)
1409	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1410	}
1411	else
1412	{
1413	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1414	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1415	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1416	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1417
1418	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1419	* not be executed. See hmQueueInvlPage() where it is commented
1420	* out. Support individual entry flushing someday. */
1421	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1422	{
1423	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1424	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1425	{
1426	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1427	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1428	}
1429	else
1430	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1431	}
1432	else
1433	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1434	}
1435
1436	pVCpu->hm.s.TlbShootdown.cPages = 0;
1437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1438
1439	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1440	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1441	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1442	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1443	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1444	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1445
1446	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1447	AssertRC(rc);
1448	}
1449
1450
1451	/**
1452	* Flushes the guest TLB entry based on CPU capabilities.
1453	*
1454	* @param pVCpu Pointer to the VMCPU.
1455	*/
1456	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1457	{
1458	PVM pVM = pVCpu->CTX_SUFF(pVM);
1459	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1460	{
1461	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1462	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1463	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1464	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1465	default:
1466	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1467	break;
1468	}
1469	}
1470
1471
1472	/**
1473	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1474	* TLB entries from the host TLB before VM-entry.
1475	*
1476	* @returns VBox status code.
1477	* @param pVM Pointer to the VM.
1478	*/
1479	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1480	{
1481	/*
1482	* Determine optimal flush type for nested paging.
1483	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1484	* guest execution (see hmR3InitFinalizeR0()).
1485	*/
1486	if (pVM->hm.s.fNestedPaging)
1487	{
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1489	{
1490	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1491	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1492	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1493	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1494	else
1495	{
1496	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1497	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1498	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1499	}
1500
1501	/* Make sure the write-back cacheable memory type for EPT is supported. */
1502	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1503	{
1504	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1505	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1506	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1507	}
1508	}
1509	else
1510	{
1511	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1512	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1513	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1514	}
1515	}
1516
1517	/*
1518	* Determine optimal flush type for VPID.
1519	*/
1520	if (pVM->hm.s.vmx.fVpid)
1521	{
1522	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1523	{
1524	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1525	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1526	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1527	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1528	else
1529	{
1530	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1531	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1532	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1533	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1534	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1535	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1536	pVM->hm.s.vmx.fVpid = false;
1537	}
1538	}
1539	else
1540	{
1541	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1542	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1543	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1544	pVM->hm.s.vmx.fVpid = false;
1545	}
1546	}
1547
1548	/*
1549	* Setup the handler for flushing tagged-TLBs.
1550	*/
1551	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1552	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1553	else if (pVM->hm.s.fNestedPaging)
1554	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1555	else if (pVM->hm.s.vmx.fVpid)
1556	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1557	else
1558	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1559	return VINF_SUCCESS;
1560	}
1561
1562
1563	/**
1564	* Sets up pin-based VM-execution controls in the VMCS.
1565	*
1566	* @returns VBox status code.
1567	* @param pVM Pointer to the VM.
1568	* @param pVCpu Pointer to the VMCPU.
1569	*/
1570	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1571	{
1572	AssertPtr(pVM);
1573	AssertPtr(pVCpu);
1574
1575	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1576	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1577
1578	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1579	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1580	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1581
1582	/* Enable the VMX preemption timer. */
1583	if (pVM->hm.s.vmx.fUsePreemptTimer)
1584	{
1585	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1586	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1587	}
1588
1589	if ((val & zap) != val)
1590	{
1591	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1592	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1593	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1594	}
1595
1596	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1597	AssertRCReturn(rc, rc);
1598
1599	/* Update VCPU with the currently set pin-based VM-execution controls. */
1600	pVCpu->hm.s.vmx.u32PinCtls = val;
1601	return rc;
1602	}
1603
1604
1605	/**
1606	* Sets up processor-based VM-execution controls in the VMCS.
1607	*
1608	* @returns VBox status code.
1609	* @param pVM Pointer to the VM.
1610	* @param pVMCPU Pointer to the VMCPU.
1611	*/
1612	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1613	{
1614	AssertPtr(pVM);
1615	AssertPtr(pVCpu);
1616
1617	int rc = VERR_INTERNAL_ERROR_5;
1618	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1619	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1620
1621	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1622	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1623	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1624	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1625	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1626	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1627	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1628
1629	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1630	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1631	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1632	{
1633	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1634	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1635	}
1636
1637	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1638	if (!pVM->hm.s.fNestedPaging)
1639	{
1640	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1641	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1644	}
1645
1646	/* Use TPR shadowing if supported by the CPU. */
1647	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1648	{
1649	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1650	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1651	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1652	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1653	AssertRCReturn(rc, rc);
1654
1655	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1656	/* CR8 writes causes a VM-exit based on TPR threshold. */
1657	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1658	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1659	}
1660	else
1661	{
1662	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1663	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1664	}
1665
1666	/* Use MSR-bitmaps if supported by the CPU. */
1667	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1668	{
1669	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1670
1671	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1672	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1673	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1674	AssertRCReturn(rc, rc);
1675
1676	/*
1677	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1678	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1679	*/
1680	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1681	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1682	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1683	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1684	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1685	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1686	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1687	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1688	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1689	}
1690
1691	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1692	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1693	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1694
1695	if ((val & zap) != val)
1696	{
1697	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1698	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1699	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1700	}
1701
1702	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1703	AssertRCReturn(rc, rc);
1704
1705	/* Update VCPU with the currently set processor-based VM-execution controls. */
1706	pVCpu->hm.s.vmx.u32ProcCtls = val;
1707
1708	/*
1709	* Secondary processor-based VM-execution controls.
1710	*/
1711	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1712	{
1713	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1714	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1715
1716	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1717	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1718
1719	if (pVM->hm.s.fNestedPaging)
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1721	else
1722	{
1723	/*
1724	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1725	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1726	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1727	*/
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1729	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1730	}
1731
1732	if (pVM->hm.s.vmx.fVpid)
1733	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1734
1735	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1736	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1737
1738	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1739	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1740	* done dynamically. */
1741	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1742	{
1743	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1744	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1746	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1747	AssertRCReturn(rc, rc);
1748	}
1749
1750	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1751	{
1752	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1753	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1754	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1755	}
1756
1757	if ((val & zap) != val)
1758	{
1759	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1760	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1761	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1762	}
1763
1764	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1765	AssertRCReturn(rc, rc);
1766
1767	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1768	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1769	}
1770
1771	return VINF_SUCCESS;
1772	}
1773
1774
1775	/**
1776	* Sets up miscellaneous (everything other than Pin & Processor-based
1777	* VM-execution) control fields in the VMCS.
1778	*
1779	* @returns VBox status code.
1780	* @param pVM Pointer to the VM.
1781	* @param pVCpu Pointer to the VMCPU.
1782	*/
1783	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1784	{
1785	AssertPtr(pVM);
1786	AssertPtr(pVCpu);
1787
1788	int rc = VERR_GENERAL_FAILURE;
1789
1790	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1791	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1792
1793	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1794
1795	/*
1796	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1797	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1798	* We thus use the exception bitmap to control it rather than use both.
1799	*/
1800	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1801	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1802
1803	/** @todo Explore possibility of using IO-bitmaps. */
1804	/* All IO & IOIO instructions cause VM-exits. */
1805	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1806	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1807
1808	/* Setup MSR autoloading/autostoring. */
1809	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1810	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1811	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1812	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1813	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1814	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1815
1816	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1817	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1818	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1820
1821	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1823
1824	/* Setup debug controls */
1825	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1826	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1827	AssertRCReturn(rc, rc);
1828	return rc;
1829	}
1830
1831
1832	/**
1833	* Sets up the initial exception bitmap in the VMCS based on static conditions
1834	* (i.e. conditions that cannot ever change at runtime).
1835	*
1836	* @returns VBox status code.
1837	* @param pVM Pointer to the VM.
1838	* @param pVCpu Pointer to the VMCPU.
1839	*/
1840	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1841	{
1842	AssertPtr(pVM);
1843	AssertPtr(pVCpu);
1844
1845	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1846
1847	uint32_t u32XcptBitmap = 0;
1848
1849	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1850	if (!pVM->hm.s.fNestedPaging)
1851	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1852
1853	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1854	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1855	AssertRCReturn(rc, rc);
1856	return rc;
1857	}
1858
1859
1860	/**
1861	* Sets up the initial guest-state mask. The guest-state mask is consulted
1862	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1863	* for the nested virtualization case (as it would cause a VM-exit).
1864	*
1865	* @param pVCpu Pointer to the VMCPU.
1866	*/
1867	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1868	{
1869	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1870	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1871	return VINF_SUCCESS;
1872	}
1873
1874
1875	/**
1876	* Does per-VM VT-x initialization.
1877	*
1878	* @returns VBox status code.
1879	* @param pVM Pointer to the VM.
1880	*/
1881	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1882	{
1883	LogFlowFunc(("pVM=%p\n", pVM));
1884
1885	int rc = hmR0VmxStructsAlloc(pVM);
1886	if (RT_FAILURE(rc))
1887	{
1888	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1889	return rc;
1890	}
1891
1892	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1893	{
1894	PVMCPU pVCpu = &pVM->aCpus[i];
1895
1896	/* Current guest paging mode. */
1897	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1898	}
1899
1900	return VINF_SUCCESS;
1901	}
1902
1903
1904	/**
1905	* Does per-VM VT-x termination.
1906	*
1907	* @returns VBox status code.
1908	* @param pVM Pointer to the VM.
1909	*/
1910	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1911	{
1912	LogFlowFunc(("pVM=%p\n", pVM));
1913
1914	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1915	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1916	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1917	#endif
1918	hmR0VmxStructsFree(pVM);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Sets up the VM for execution under VT-x.
1925	* This function is only called once per-VM during initalization.
1926	*
1927	* @returns VBox status code.
1928	* @param pVM Pointer to the VM.
1929	*/
1930	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1931	{
1932	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1933	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1934
1935	LogFlowFunc(("pVM=%p\n", pVM));
1936
1937	/*
1938	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1939	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1940	*/
1941	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1942	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1943	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1944	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1945	{
1946	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1947	return VERR_INTERNAL_ERROR;
1948	}
1949
1950	/* Initialize these always, see hmR3InitFinalizeR0().*/
1951	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1952	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1953
1954	/* Setup the tagged-TLB flush handlers. */
1955	int rc = hmR0VmxSetupTaggedTlb(pVM);
1956	if (RT_FAILURE(rc))
1957	{
1958	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1959	return rc;
1960	}
1961
1962	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1963	{
1964	PVMCPU pVCpu = &pVM->aCpus[i];
1965	AssertPtr(pVCpu);
1966	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1967
1968	/* Set revision dword at the beginning of the VMCS structure. */
1969	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1970
1971	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1972	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1973	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1974	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1975
1976	/* Load this VMCS as the current VMCS. */
1977	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1978	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1979	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1980
1981	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1982	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1983	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1984
1985	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1986	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1987	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1988
1989	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1990	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1991	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1992
1993	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1994	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1995	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1996
1997	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000
2001	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2002	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
2003	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2004	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2005	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2006	#endif
2007
2008	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2009	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2010	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2011	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2012
2013	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2014	}
2015
2016	return VINF_SUCCESS;
2017	}
2018
2019
2020	/**
2021	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2022	* the VMCS.
2023	*
2024	* @returns VBox status code.
2025	* @param pVM Pointer to the VM.
2026	* @param pVCpu Pointer to the VMCPU.
2027	*/
2028	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2029	{
2030	RTCCUINTREG uReg = ASMGetCR0();
2031	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2032
2033	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2034	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2035	if (HMVMX_IS_64BIT_HOST_MODE())
2036	{
2037	uint64_t uReg = hmR0Get64bitCR3();
2038	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2039	}
2040	else
2041	#endif
2042	{
2043	uReg = ASMGetCR3();
2044	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2045	}
2046
2047	uReg = ASMGetCR4();
2048	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2049	AssertRCReturn(rc, rc);
2050	return rc;
2051	}
2052
2053
2054	/**
2055	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2056	* the host-state area in the VMCS.
2057	*
2058	* @returns VBox status code.
2059	* @param pVM Pointer to the VM.
2060	* @param pVCpu Pointer to the VMCPU.
2061	*/
2062	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2063	{
2064	int rc = VERR_INTERNAL_ERROR_5;
2065	RTSEL uSelCS = 0;
2066	RTSEL uSelSS = 0;
2067	RTSEL uSelDS = 0;
2068	RTSEL uSelES = 0;
2069	RTSEL uSelFS = 0;
2070	RTSEL uSelGS = 0;
2071	RTSEL uSelTR = 0;
2072
2073	/*
2074	* Host Selector registers.
2075	*/
2076	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2077	if (HMVMX_IS_64BIT_HOST_MODE())
2078	{
2079	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2080	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2081	}
2082	else
2083	{
2084	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2085	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2086	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2087	}
2088	#else
2089	uSelCS = ASMGetCS();
2090	uSelSS = ASMGetSS();
2091	#endif
2092
2093	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2094	/** @todo Verify if we have any platform that actually run with DS or ES with
2095	* RPL != 0 in kernel space. */
2096	uSelDS = 0;
2097	uSelES = 0;
2098	uSelFS = 0;
2099	uSelGS = 0;
2100	uSelTR = ASMGetTR();
2101
2102	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2103	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2104	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2105	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2106	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2107	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2108	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2109	Assert(uSelCS != 0);
2110	Assert(uSelTR != 0);
2111
2112	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2113	#if 0
2114	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2115	Assert(uSelSS != 0);
2116	#endif
2117
2118	/* Write these host selector fields into the host-state area in the VMCS. */
2119	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2120	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2121	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2122	#if 0
2123	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2124	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2125	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2126	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2127	#endif
2128	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2129	AssertRCReturn(rc, rc);
2130
2131	/*
2132	* Host GDTR and IDTR.
2133	*/
2134	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2135	* be safe to -not- save and restore GDTR and IDTR in the assembly
2136	* code and just do it here and don't care if the limits are zapped on
2137	* VM-exit. */
2138	RTGDTR Gdtr;
2139	RT_ZERO(Gdtr);
2140	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2141	if (HMVMX_IS_64BIT_HOST_MODE())
2142	{
2143	X86XDTR64 Gtr64;
2144	X86XDTR64 Idtr64;
2145	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2146	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2147	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2148	Gdtr.cbGdt = Gdtr64.cb;
2149	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2150	}
2151	else
2152	#endif
2153	{
2154	RTIDTR Idtr;
2155	ASMGetGDTR(&Gdtr);
2156	ASMGetIDTR(&Idtr);
2157	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2158	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2159	}
2160	AssertRCReturn(rc, rc);
2161
2162	/*
2163	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2164	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2165	*/
2166	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2167	{
2168	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2169	return VERR_VMX_INVALID_HOST_STATE;
2170	}
2171
2172	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2173	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2174	if (HMVMX_IS_64BIT_HOST_MODE())
2175	{
2176	/* We need the 64-bit TR base for hybrid darwin. */
2177	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2178	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2179	}
2180	else
2181	#endif
2182	{
2183	uintptr_t uTRBase;
2184	#if HC_ARCH_BITS == 64
2185	uTRBase = X86DESC64_BASE(pDesc);
2186	#else
2187	uTRBase = X86DESC_BASE(pDesc);
2188	#endif
2189	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2190	}
2191	AssertRCReturn(rc, rc);
2192
2193	/*
2194	* Host FS base and GS base.
2195	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2196	* would take care of the bases. In 64-bit, the MSRs come into play.
2197	*/
2198	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2199	if (HMVMX_IS_64BIT_HOST_MODE())
2200	{
2201	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2202	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2203	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2204	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2205	AssertRCReturn(rc, rc);
2206	}
2207	#endif
2208	return rc;
2209	}
2210
2211
2212	/**
2213	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2214	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2215	* the host after every successful VM exit.
2216	*
2217	* @returns VBox status code.
2218	* @param pVM Pointer to the VM.
2219	* @param pVCpu Pointer to the VMCPU.
2220	*/
2221	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2222	{
2223	AssertPtr(pVCpu);
2224	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2225
2226	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2227	uint32_t cHostMsrs = 0;
2228	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2229
2230	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2231	{
2232	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2233	pHostMsr->u32Reserved = 0;
2234	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2235	if (CPUMIsGuestInLongMode(pVCpu))
2236	{
2237	/* Must match the EFER value in our 64 bits switcher. */
2238	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2239	}
2240	else
2241	#endif
2242	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2243	pHostMsr++; cHostMsrs++;
2244	}
2245
2246	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2247	if (HMVMX_IS_64BIT_HOST_MODE())
2248	{
2249	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2250	pHostMsr->u32Reserved = 0;
2251	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2252	pHostMsr++; cHostMsrs++;
2253	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2254	pHostMsr->u32Reserved = 0;
2255	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2256	pHostMsr++; cHostMsrs++;
2257	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2258	pHostMsr->u32Reserved = 0;
2259	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2260	pHostMsr++; cHostMsrs++;
2261	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2262	#if 0
2263	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2264	pMsr->u32Reserved = 0;
2265	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2266	pHostMsr++; cHostMsrs++;
2267	#endif
2268	}
2269	#endif
2270
2271	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2272	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2273	{
2274	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2275	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2276	}
2277
2278	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2279
2280	/*
2281	* Host Sysenter MSRs.
2282	*/
2283	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2284	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2285	if (HMVMX_IS_64BIT_HOST_MODE())
2286	{
2287	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2288	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2289	}
2290	else
2291	{
2292	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2293	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2294	}
2295	#elif HC_ARCH_BITS == 32
2296	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2297	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2298	#else
2299	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2300	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2301	#endif
2302	AssertRCReturn(rc, rc);
2303
2304	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2305	* hmR0VmxSetupExitCtls() !! */
2306	return rc;
2307	}
2308
2309
2310	/**
2311	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2312	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2313	* controls".
2314	*
2315	* @returns VBox status code.
2316	* @param pVCpu Pointer to the VMCPU.
2317	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2318	* out-of-sync. Make sure to update the required fields
2319	* before using them.
2320	*
2321	* @remarks No-long-jump zone!!!
2322	*/
2323	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2324	{
2325	int rc = VINF_SUCCESS;
2326	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2327	{
2328	PVM pVM = pVCpu->CTX_SUFF(pVM);
2329	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2330	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2331
2332	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2333	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2334
2335	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2336	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2337	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2338	else
2339	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2340
2341	/*
2342	* The following should not be set (since we're not in SMM mode):
2343	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2344	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2345	*/
2346
2347	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2348	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2349	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2350
2351	if ((val & zap) != val)
2352	{
2353	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2354	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2355	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2356	}
2357
2358	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2359	AssertRCReturn(rc, rc);
2360
2361	/* Update VCPU with the currently set VM-exit controls. */
2362	pVCpu->hm.s.vmx.u32EntryCtls = val;
2363	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2364	}
2365	return rc;
2366	}
2367
2368
2369	/**
2370	* Sets up the VM-exit controls in the VMCS.
2371	*
2372	* @returns VBox status code.
2373	* @param pVM Pointer to the VM.
2374	* @param pVCpu Pointer to the VMCPU.
2375	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2376	* out-of-sync. Make sure to update the required fields
2377	* before using them.
2378	*
2379	* @remarks requires EFER.
2380	*/
2381	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2382	{
2383	int rc = VINF_SUCCESS;
2384	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2385	{
2386	PVM pVM = pVCpu->CTX_SUFF(pVM);
2387	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2388	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2389
2390	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2391	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2392
2393	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2394	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2395	if (HMVMX_IS_64BIT_HOST_MODE())
2396	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2397	else
2398	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2399	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2400	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2401	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2402	else
2403	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2404	#endif
2405
2406	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2407	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2408
2409	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2410	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2411	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2412	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2413	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2414
2415	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2416	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2417
2418	if ((val & zap) != val)
2419	{
2420	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2421	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2422	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2423	}
2424
2425	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2426	AssertRCReturn(rc, rc);
2427
2428	/* Update VCPU with the currently set VM-exit controls. */
2429	pVCpu->hm.s.vmx.u32ExitCtls = val;
2430	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2431	}
2432	return rc;
2433	}
2434
2435
2436	/**
2437	* Loads the guest APIC and related state.
2438	*
2439	* @returns VBox status code.
2440	* @param pVM Pointer to the VM.
2441	* @param pVCpu Pointer to the VMCPU.
2442	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2443	* out-of-sync. Make sure to update the required fields
2444	* before using them.
2445	*/
2446	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2447	{
2448	int rc = VINF_SUCCESS;
2449	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2450	{
2451	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2452	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2453	{
2454	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2455
2456	bool fPendingIntr = false;
2457	uint8_t u8GuestTpr = 0;
2458	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2459	AssertRCReturn(rc, rc);
2460
2461	/*
2462	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2463	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2464	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2465	* the interrupt when we VM-exit for other reasons.
2466	*/
2467	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2468	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2469	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2470	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2471
2472	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2473	AssertRCReturn(rc, rc);
2474
2475	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2476	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2477	{
2478	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2479	pMixedCtx->msrLSTAR = u8GuestTpr;
2480	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2481	{
2482	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2483	if (fPendingIntr)
2484	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2485	else
2486	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2487	}
2488	}
2489	}
2490
2491	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2492	}
2493	return rc;
2494	}
2495
2496
2497	/**
2498	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2499	*
2500	* @returns
2501	* @param pVCpu Pointer to the VMCPU.
2502	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2503	* out-of-sync. Make sure to update the required fields
2504	* before using them.
2505	*
2506	* @remarks No-long-jump zone!!!
2507	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2508	*/
2509	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2510	{
2511	/*
2512	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2513	* inhibit interrupts or clear any existing interrupt-inhibition.
2514	*/
2515	uint32_t uIntrState = 0;
2516	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2517	{
2518	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2519	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2520	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2521	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2522	{
2523	/*
2524	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2525	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2526	*/
2527	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2528	}
2529	else if (pMixedCtx->eflags.Bits.u1IF)
2530	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2531	else
2532	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2533	}
2534	return uIntrState;
2535	}
2536
2537
2538	/**
2539	* Loads the guest's interruptibility-state into the guest-state area in the
2540	* VMCS.
2541	*
2542	* @returns VBox status code.
2543	* @param pVCpu Pointer to the VMCPU.
2544	* @param uIntrState The interruptibility-state to set.
2545	*/
2546	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2547	{
2548	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2549	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2550	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2551	AssertRCReturn(rc, rc);
2552	return rc;
2553	}
2554
2555
2556	/**
2557	* Loads the guest's RIP into the guest-state area in the VMCS.
2558	*
2559	* @returns VBox status code.
2560	* @param pVCpu Pointer to the VMCPU.
2561	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2562	* out-of-sync. Make sure to update the required fields
2563	* before using them.
2564	*
2565	* @remarks No-long-jump zone!!!
2566	*/
2567	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2568	{
2569	int rc = VINF_SUCCESS;
2570	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2571	{
2572	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2573	AssertRCReturn(rc, rc);
2574	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2575	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2576	}
2577	return rc;
2578	}
2579
2580
2581	/**
2582	* Loads the guest's RSP into the guest-state area in the VMCS.
2583	*
2584	* @returns VBox status code.
2585	* @param pVCpu Pointer to the VMCPU.
2586	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2587	* out-of-sync. Make sure to update the required fields
2588	* before using them.
2589	*
2590	* @remarks No-long-jump zone!!!
2591	*/
2592	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2593	{
2594	int rc = VINF_SUCCESS;
2595	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2596	{
2597	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2598	AssertRCReturn(rc, rc);
2599	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2600	}
2601	return rc;
2602	}
2603
2604
2605	/**
2606	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2607	*
2608	* @returns VBox status code.
2609	* @param pVCpu Pointer to the VMCPU.
2610	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2611	* out-of-sync. Make sure to update the required fields
2612	* before using them.
2613	*
2614	* @remarks No-long-jump zone!!!
2615	*/
2616	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2617	{
2618	int rc = VINF_SUCCESS;
2619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2620	{
2621	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2622	Let us assert it as such and use native-width VMWRITE. */
2623	X86RFLAGS uRFlags = pMixedCtx->rflags;
2624	Assert(uRFlags.u64 >> 32 == 0);
2625	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2626	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2627
2628	/*
2629	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2630	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2631	*/
2632	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2633	{
2634	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2635	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2636	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2637	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2638	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2639	}
2640
2641	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2642	AssertRCReturn(rc, rc);
2643
2644	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2645	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2646	}
2647	return rc;
2648	}
2649
2650
2651	/**
2652	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2653	*
2654	* @returns VBox status code.
2655	* @param pVCpu Pointer to the VMCPU.
2656	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2657	* out-of-sync. Make sure to update the required fields
2658	* before using them.
2659	*
2660	* @remarks No-long-jump zone!!!
2661	*/
2662	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2663	{
2664	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2665	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2666	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2667	return rc;
2668	}
2669
2670
2671	/**
2672	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2673	* in the VMCS.
2674	*
2675	* @returns VBox status code.
2676	* @param pVM Pointer to the VM.
2677	* @param pVCpu Pointer to the VMCPU.
2678	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2679	* out-of-sync. Make sure to update the required fields
2680	* before using them.
2681	*
2682	* @remarks No-long-jump zone!!!
2683	*/
2684	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2685	{
2686	int rc = VINF_SUCCESS;
2687	PVM pVM = pVCpu->CTX_SUFF(pVM);
2688
2689	/*
2690	* Guest CR0.
2691	* Guest FPU.
2692	*/
2693	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2694	{
2695	uint64_t u64GuestCR0 = pCtx->cr0;
2696
2697	/* The guest's view (read access) of its CR0 is unblemished. */
2698	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2699	AssertRCReturn(rc, rc);
2700	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2701
2702	/* Setup VT-x's view of the guest CR0. */
2703	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2704	if (pVM->hm.s.fNestedPaging)
2705	{
2706	if (CPUMIsGuestPagingEnabledEx(pCtx))
2707	{
2708	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2709	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2710	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2711	}
2712	else
2713	{
2714	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2715	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2716	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2717	}
2718
2719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2720	AssertRCReturn(rc, rc);
2721	}
2722	else
2723	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2724
2725	/*
2726	* Guest FPU bits.
2727	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2728	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2729	*/
2730	u64GuestCR0 \|= X86_CR0_NE;
2731	bool fInterceptNM = false;
2732	if (CPUMIsGuestFPUStateActive(pVCpu))
2733	{
2734	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2735	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2736	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2737	}
2738	else
2739	{
2740	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2741	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2742	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2743	}
2744
2745	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2746	bool fInterceptMF = false;
2747	if (!(pCtx->cr0 & X86_CR0_NE))
2748	fInterceptMF = true;
2749
2750	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2751	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2752	{
2753	Assert(PDMVmmDevHeapIsEnabled(pVM));
2754	Assert(pVM->hm.s.vmx.pRealModeTSS);
2755	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2756	fInterceptNM = true;
2757	fInterceptMF = true;
2758	}
2759	else
2760	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2761
2762	if (fInterceptNM)
2763	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2764	else
2765	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2766
2767	if (fInterceptMF)
2768	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2769	else
2770	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2771
2772	/* Additional intercepts for debugging, define these yourself explicitly. */
2773	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2775	\| RT_BIT(X86_XCPT_DB)
2776	\| RT_BIT(X86_XCPT_DE)
2777	\| RT_BIT(X86_XCPT_NM)
2778	\| RT_BIT(X86_XCPT_UD)
2779	\| RT_BIT(X86_XCPT_NP)
2780	\| RT_BIT(X86_XCPT_SS)
2781	\| RT_BIT(X86_XCPT_GP)
2782	\| RT_BIT(X86_XCPT_PF)
2783	\| RT_BIT(X86_XCPT_MF);
2784	#elif defined(VBOX_ALWAYS_TRAP_PF)
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2786	#endif
2787
2788	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2789
2790	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2791	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2792	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2793	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2794	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2795	else
2796	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2797
2798	u64GuestCR0 \|= uSetCR0;
2799	u64GuestCR0 &= uZapCR0;
2800	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2801
2802	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2803	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2804	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2805	Log(("Load: VMX_VMCS_GUEST_CR0=%RGv (uSetCR0=%#RGv uZapCR0=%#RGv)\n", u64GuestCR0, uSetCR0, uZapCR0));
2806
2807	/*
2808	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2809	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2810	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2811	*/
2812	uint64_t u64CR0Mask = 0;
2813	u64CR0Mask = X86_CR0_PE
2814	\| X86_CR0_NE
2815	\| X86_CR0_WP
2816	\| X86_CR0_PG
2817	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2818	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2819	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2820	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2821	u64CR0Mask &= ~X86_CR0_PE;
2822	/* Enable this later. */
2823	if (pVM->hm.s.fNestedPaging)
2824	u64CR0Mask &= ~X86_CR0_WP;
2825
2826	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2827	if (fInterceptNM)
2828	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2829	else
2830	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2831
2832	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2833	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2834	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2835	AssertRCReturn(rc, rc);
2836
2837	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2838	}
2839
2840	/*
2841	* Guest CR2.
2842	* It's always loaded in the assembler code. Nothing to do here.
2843	*/
2844
2845	/*
2846	* Guest CR3.
2847	*/
2848	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2849	{
2850	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2851	if (pVM->hm.s.fNestedPaging)
2852	{
2853	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2854
2855	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2856	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2857	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2858	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2859
2860	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2861	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2862	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2863
2864	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2865	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2866	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2867	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2868
2869	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2870	AssertRCReturn(rc, rc);
2871	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2872
2873	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2874	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2875	{
2876	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2877	if (CPUMIsGuestInPAEModeEx(pCtx))
2878	{
2879	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2880	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2881	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2882	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2883	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2884	AssertRCReturn(rc, rc);
2885	}
2886
2887	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2888	have Unrestricted Execution to handle the guest when it's not using paging. */
2889	GCPhysGuestCR3 = pCtx->cr3;
2890	}
2891	else
2892	{
2893	/*
2894	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2895	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2896	* EPT takes care of translating it to host-physical addresses.
2897	*/
2898	RTGCPHYS GCPhys;
2899	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2900	Assert(PDMVmmDevHeapIsEnabled(pVM));
2901
2902	/* We obtain it here every time as the guest could have relocated this PCI region. */
2903	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2904	AssertRCReturn(rc, rc);
2905
2906	GCPhysGuestCR3 = GCPhys;
2907	}
2908	}
2909	else
2910	{
2911	/* Non-nested paging case, just use the hypervisor's CR3. */
2912	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2913	}
2914
2915	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2916	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2917	AssertRCReturn(rc, rc);
2918
2919	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2920	}
2921
2922	/*
2923	* Guest CR4.
2924	*/
2925	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2926	{
2927	uint32_t u64GuestCR4 = pCtx->cr4;
2928	Assert(!(pCtx->cr4 >> 32));
2929
2930	/* The guest's view of its CR4 is unblemished. */
2931	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2932	AssertRCReturn(rc, rc);
2933	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2934
2935	/* Setup VT-x's view of the guest CR4. */
2936	/*
2937	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2938	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2939	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2940	*/
2941	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2942	{
2943	Assert(pVM->hm.s.vmx.pRealModeTSS);
2944	Assert(PDMVmmDevHeapIsEnabled(pVM));
2945	u64GuestCR4 &= ~X86_CR4_VME;
2946	}
2947
2948	if (pVM->hm.s.fNestedPaging)
2949	{
2950	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2951	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2952	{
2953	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2954	u64GuestCR4 \|= X86_CR4_PSE;
2955	/* Our identity mapping is a 32 bits page directory. */
2956	u64GuestCR4 &= ~X86_CR4_PAE;
2957	}
2958	/* else use guest CR4.*/
2959	}
2960	else
2961	{
2962	/*
2963	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2964	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2965	*/
2966	switch (pVCpu->hm.s.enmShadowMode)
2967	{
2968	case PGMMODE_REAL: /* Real-mode. */
2969	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2970	case PGMMODE_32_BIT: /* 32-bit paging. */
2971	{
2972	u64GuestCR4 &= ~X86_CR4_PAE;
2973	break;
2974	}
2975
2976	case PGMMODE_PAE: /* PAE paging. */
2977	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2978	{
2979	u64GuestCR4 \|= X86_CR4_PAE;
2980	break;
2981	}
2982
2983	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2984	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2985	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2986	break;
2987	#endif
2988	default:
2989	AssertFailed();
2990	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2991	}
2992	}
2993
2994	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2995	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2996	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2997	u64GuestCR4 \|= uSetCR4;
2998	u64GuestCR4 &= uZapCR4;
2999
3000	/* Write VT-x's view of the guest CR4 into the VMCS. */
3001	Log(("Load: VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
3002	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
3003
3004	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3005	uint64_t u64CR4Mask = 0;
3006	u64CR4Mask = X86_CR4_VME
3007	\| X86_CR4_PAE
3008	\| X86_CR4_PGE
3009	\| X86_CR4_PSE
3010	\| X86_CR4_VMXE;
3011	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3012	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3013	AssertRCReturn(rc, rc);
3014
3015	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3016	}
3017	return rc;
3018	}
3019
3020
3021	/**
3022	* Loads the guest debug registers into the guest-state area in the VMCS.
3023	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3024	*
3025	* @returns VBox status code.
3026	* @param pVCpu Pointer to the VMCPU.
3027	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3028	* out-of-sync. Make sure to update the required fields
3029	* before using them.
3030	*
3031	* @remarks No-long-jump zone!!!
3032	*/
3033	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3034	{
3035	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3036	return VINF_SUCCESS;
3037
3038	#ifdef VBOX_STRICT
3039	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3040	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3041	{
3042	Assert((pMixedCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3043	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3044	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3045	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3046	}
3047	#endif
3048
3049	int rc = VERR_INTERNAL_ERROR_5;
3050	PVM pVM = pVCpu->CTX_SUFF(pVM);
3051	bool fInterceptDB = false;
3052	bool fInterceptMovDRx = false;
3053	if (DBGFIsStepping(pVCpu))
3054	{
3055	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3056	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3057	{
3058	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3059	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3060	AssertRCReturn(rc, rc);
3061	Assert(fInterceptDB == false);
3062	}
3063	else
3064	fInterceptDB = true;
3065	}
3066
3067	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3068	{
3069	if (!CPUMIsHyperDebugStateActive(pVCpu))
3070	{
3071	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3072	AssertRC(rc);
3073	}
3074	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3075	fInterceptMovDRx = true;
3076	}
3077	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3078	{
3079	if (!CPUMIsGuestDebugStateActive(pVCpu))
3080	{
3081	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3082	AssertRC(rc);
3083	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3084	}
3085	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3086	Assert(fInterceptMovDRx == false);
3087	}
3088	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3089	{
3090	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3091	fInterceptMovDRx = true;
3092	}
3093
3094	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3095	if (fInterceptDB)
3096	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3097	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3098	{
3099	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3100	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3101	#endif
3102	}
3103
3104	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3105	if (fInterceptMovDRx)
3106	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3107	else
3108	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3109
3110	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3111	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3112
3113	/* The guest's view of its DR7 is unblemished. */
3114	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3115
3116	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3117	return rc;
3118	}
3119
3120
3121	#ifdef VBOX_STRICT
3122	/**
3123	* Strict function to validate segment registers.
3124	*/
3125	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3126	{
3127	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3128	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3129	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3130	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3131	{
3132	/* Protected mode checks */
3133	/* CS */
3134	Assert(pCtx->cs.Attr.n.u1Present);
3135	Assert(!(pCtx->cs.Attr.u & 0xf00));
3136	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3137	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3138	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3139	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3140	\|\| (pCtx->cs.Attr.n.u1Granularity));
3141	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3142	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3143	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3144	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3145	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3146	else
3147	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3148	/* SS */
3149	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3150	{
3151	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3152	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3153	Assert(pCtx->ss.Attr.n.u1Present);
3154	Assert(!(pCtx->ss.Attr.u & 0xf00));
3155	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3156	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3157	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3158	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3159	\|\| (pCtx->ss.Attr.n.u1Granularity));
3160	}
3161	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3162	/* CR0 might not be up-to-date here always, hence disabled. */
3163	#if 0
3164	if (!pCtx->cr0 & X86_CR0_PE)
3165	Assert(!pCtx->ss.Attr.n.u2Dpl);
3166	#endif
3167	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3168	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3169	{
3170	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3171	Assert(pCtx->ds.Attr.n.u1Present);
3172	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3173	Assert(!(pCtx->ds.Attr.u & 0xf00));
3174	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3175	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3176	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3177	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3178	\|\| (pCtx->ds.Attr.n.u1Granularity));
3179	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3180	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3181	}
3182	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3183	{
3184	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3185	Assert(pCtx->es.Attr.n.u1Present);
3186	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3187	Assert(!(pCtx->es.Attr.u & 0xf00));
3188	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3189	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3190	\|\| !(pCtx->es.Attr.n.u1Granularity));
3191	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3192	\|\| (pCtx->es.Attr.n.u1Granularity));
3193	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3194	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3195	}
3196	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3197	{
3198	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3199	Assert(pCtx->fs.Attr.n.u1Present);
3200	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3201	Assert(!(pCtx->fs.Attr.u & 0xf00));
3202	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3203	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3204	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3205	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3206	\|\| (pCtx->fs.Attr.n.u1Granularity));
3207	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3208	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3209	}
3210	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3211	{
3212	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3213	Assert(pCtx->gs.Attr.n.u1Present);
3214	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3215	Assert(!(pCtx->gs.Attr.u & 0xf00));
3216	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3217	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3218	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3219	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3220	\|\| (pCtx->gs.Attr.n.u1Granularity));
3221	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3222	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3223	}
3224	/* 64-bit capable CPUs. */
3225	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3226	Assert(!(pCtx->cs.u64Base >> 32));
3227	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3228	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3229	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3230	# endif
3231	}
3232	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3233	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3234	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3235	{
3236	/* Real and v86 mode checks. */
3237	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3238	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3239	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3240	{
3241	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3242	}
3243	else
3244	{
3245	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3246	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3247	}
3248
3249	/* CS */
3250	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3251	Assert(pCtx->cs.u32Limit == 0xffff);
3252	Assert(u32CSAttr == 0xf3);
3253	/* SS */
3254	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3255	Assert(pCtx->ss.u32Limit == 0xffff);
3256	Assert(u32SSAttr == 0xf3);
3257	/* DS */
3258	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3259	Assert(pCtx->ds.u32Limit == 0xffff);
3260	Assert(u32DSAttr == 0xf3);
3261	/* ES */
3262	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3263	Assert(pCtx->es.u32Limit == 0xffff);
3264	Assert(u32ESAttr == 0xf3);
3265	/* FS */
3266	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3267	Assert(pCtx->fs.u32Limit == 0xffff);
3268	Assert(u32FSAttr == 0xf3);
3269	/* GS */
3270	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3271	Assert(pCtx->gs.u32Limit == 0xffff);
3272	Assert(u32GSAttr == 0xf3);
3273	/* 64-bit capable CPUs. */
3274	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3275	Assert(!(pCtx->cs.u64Base >> 32));
3276	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3277	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3278	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3279	# endif
3280	}
3281	}
3282	#endif /* VBOX_STRICT */
3283
3284
3285	/**
3286	* Writes a guest segment register into the guest-state area in the VMCS.
3287	*
3288	* @returns VBox status code.
3289	* @param pVCpu Pointer to the VMCPU.
3290	* @param idxSel Index of the selector in the VMCS.
3291	* @param idxLimit Index of the segment limit in the VMCS.
3292	* @param idxBase Index of the segment base in the VMCS.
3293	* @param idxAccess Index of the access rights of the segment in the VMCS.
3294	* @param pSelReg Pointer to the segment selector.
3295	* @param pCtx Pointer to the guest-CPU context.
3296	*
3297	* @remarks No-long-jump zone!!!
3298	*/
3299	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3300	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3301	{
3302	int rc;
3303	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3304	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3305	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3306	AssertRCReturn(rc, rc);
3307
3308	uint32_t u32Access = pSelReg->Attr.u;
3309	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3310	{
3311	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3312	u32Access = 0xf3;
3313	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3314	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3315	}
3316	else
3317	{
3318	/*
3319	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3320	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3321	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3322	* loaded in protected-mode have their attribute as 0.
3323	*/
3324	if (!u32Access)
3325	u32Access = HMVMX_SEL_UNUSABLE;
3326	}
3327
3328	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3329	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3330	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3331
3332	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3333	AssertRCReturn(rc, rc);
3334	return rc;
3335	}
3336
3337
3338	/**
3339	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3340	* into the guest-state area in the VMCS.
3341	*
3342	* @returns VBox status code.
3343	* @param pVM Pointer to the VM.
3344	* @param pVCPU Pointer to the VMCPU.
3345	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3346	* out-of-sync. Make sure to update the required fields
3347	* before using them.
3348	*
3349	* @remarks No-long-jump zone!!!
3350	*/
3351	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3352	{
3353	int rc = VERR_INTERNAL_ERROR_5;
3354	PVM pVM = pVCpu->CTX_SUFF(pVM);
3355
3356	/*
3357	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3358	*/
3359	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3360	{
3361	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3362	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3363	{
3364	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3365	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3366	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3367	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3368	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3369	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3370	}
3371
3372	#ifdef VBOX_WITH_REM
3373	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3374	{
3375	Assert(pVM->hm.s.vmx.pRealModeTSS);
3376	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3377	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3378	{
3379	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3380	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3381	&& enmGuestMode >= PGMMODE_PROTECTED)
3382	{
3383	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3384	in real-mode (e.g. OpenBSD 4.0) */
3385	REMFlushTBs(pVM);
3386	Log(("Load: Switch to protected mode detected!\n"));
3387	}
3388	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3389	}
3390	}
3391	#endif
3392	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3393	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3394	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3395	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3396	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3397	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3398	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3399	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3400	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3401	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3402	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3403	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3404	AssertRCReturn(rc, rc);
3405
3406	#ifdef VBOX_STRICT
3407	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3408	#endif
3409	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3410	}
3411
3412	/*
3413	* Guest TR.
3414	*/
3415	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3416	{
3417	/*
3418	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3419	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3420	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3421	*/
3422	uint16_t u16Sel = 0;
3423	uint32_t u32Limit = 0;
3424	uint64_t u64Base = 0;
3425	uint32_t u32AccessRights = 0;
3426
3427	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3428	{
3429	u16Sel = pMixedCtx->tr.Sel;
3430	u32Limit = pMixedCtx->tr.u32Limit;
3431	u64Base = pMixedCtx->tr.u64Base;
3432	u32AccessRights = pMixedCtx->tr.Attr.u;
3433	}
3434	else
3435	{
3436	Assert(pVM->hm.s.vmx.pRealModeTSS);
3437	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3438
3439	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3440	RTGCPHYS GCPhys;
3441	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3442	AssertRCReturn(rc, rc);
3443
3444	X86DESCATTR DescAttr;
3445	DescAttr.u = 0;
3446	DescAttr.n.u1Present = 1;
3447	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3448
3449	u16Sel = 0;
3450	u32Limit = HM_VTX_TSS_SIZE;
3451	u64Base = GCPhys; /* in real-mode phys = virt. */
3452	u32AccessRights = DescAttr.u;
3453	}
3454
3455	/* Validate. */
3456	Assert(!(u16Sel & RT_BIT(2)));
3457	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3458	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3459	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3460	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3461	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3462	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3463	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3464	Assert( (u32Limit & 0xfff) == 0xfff
3465	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3466	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3467	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3468
3469	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3470	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3471	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3472	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3473	AssertRCReturn(rc, rc);
3474
3475	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3476	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3477	}
3478
3479	/*
3480	* Guest GDTR.
3481	*/
3482	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3483	{
3484	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3485	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3486	AssertRCReturn(rc, rc);
3487
3488	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3489	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3490	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3491	}
3492
3493	/*
3494	* Guest LDTR.
3495	*/
3496	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3497	{
3498	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3499	uint32_t u32Access = 0;
3500	if (!pMixedCtx->ldtr.Attr.u)
3501	u32Access = HMVMX_SEL_UNUSABLE;
3502	else
3503	u32Access = pMixedCtx->ldtr.Attr.u;
3504
3505	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3506	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3507	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3508	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3509	AssertRCReturn(rc, rc);
3510
3511	/* Validate. */
3512	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3513	{
3514	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3515	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3516	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3517	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3518	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3519	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3520	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3521	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3522	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3523	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3524	}
3525
3526	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3527	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3528	}
3529
3530	/*
3531	* Guest IDTR.
3532	*/
3533	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3534	{
3535	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3536	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3537	AssertRCReturn(rc, rc);
3538
3539	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3540	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3541	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3542	}
3543
3544	return VINF_SUCCESS;
3545	}
3546
3547
3548	/**
3549	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3550	* areas. These MSRs will automatically be loaded to the host CPU on every
3551	* successful VM entry and stored from the host CPU on every successful VM exit.
3552	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3553	*
3554	* @returns VBox status code.
3555	* @param pVCpu Pointer to the VMCPU.
3556	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3557	* out-of-sync. Make sure to update the required fields
3558	* before using them.
3559	*
3560	* @remarks No-long-jump zone!!!
3561	*/
3562	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3563	{
3564	AssertPtr(pVCpu);
3565	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3566
3567	/*
3568	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3569	*/
3570	int rc = VINF_SUCCESS;
3571	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3572	{
3573	PVM pVM = pVCpu->CTX_SUFF(pVM);
3574	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3575	uint32_t cGuestMsrs = 0;
3576
3577	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3578	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3579	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3580	if (fSupportsNX \|\| fSupportsLongMode)
3581	{
3582	/** @todo support save IA32_EFER, i.e.
3583	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3584	* guest EFER need not be part of the VM-entry MSR-load area. */
3585	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3586	pGuestMsr->u32Reserved = 0;
3587	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3588	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3589	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3590	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3591	pGuestMsr++; cGuestMsrs++;
3592	if (fSupportsLongMode)
3593	{
3594	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3595	pGuestMsr->u32Reserved = 0;
3596	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3597	pGuestMsr++; cGuestMsrs++;
3598	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3599	pGuestMsr->u32Reserved = 0;
3600	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3601	pGuestMsr++; cGuestMsrs++;
3602	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3603	pGuestMsr->u32Reserved = 0;
3604	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3605	pGuestMsr++; cGuestMsrs++;
3606	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3607	#if 0
3608	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3609	pGuestMsr->u32Reserved = 0;
3610	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3611	pGuestMsr++; cGuestMsrs++;
3612	#endif
3613	}
3614	}
3615
3616	/*
3617	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3618	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3619	*/
3620	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3621	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3622	{
3623	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3624	pGuestMsr->u32Reserved = 0;
3625	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3626	AssertRCReturn(rc, rc);
3627	pGuestMsr++; cGuestMsrs++;
3628	}
3629
3630	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3631	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3632	{
3633	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3634	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3635	}
3636
3637	/* Update the VCPU's copy of the guest MSR count. */
3638	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3639	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3640	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3641	AssertRCReturn(rc, rc);
3642
3643	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3644	}
3645
3646	/*
3647	* Guest Sysenter MSRs.
3648	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3649	* VM-exits on WRMSRs for these MSRs.
3650	*/
3651	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3652	{
3653	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3654	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3655	}
3656	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3657	{
3658	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3659	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3660	}
3661	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3662	{
3663	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3664	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3665	}
3666	AssertRCReturn(rc, rc);
3667
3668	return rc;
3669	}
3670
3671
3672	/**
3673	* Loads the guest activity state into the guest-state area in the VMCS.
3674	*
3675	* @returns VBox status code.
3676	* @param pVCpu Pointer to the VMCPU.
3677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3678	* out-of-sync. Make sure to update the required fields
3679	* before using them.
3680	*
3681	* @remarks No-long-jump zone!!!
3682	*/
3683	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3684	{
3685	/** @todo See if we can make use of other states, e.g.
3686	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3687	int rc = VINF_SUCCESS;
3688	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3689	{
3690	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3691	AssertRCReturn(rc, rc);
3692	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3693	}
3694	return rc;
3695	}
3696
3697
3698	/**
3699	* Sets up the appropriate function to run guest code.
3700	*
3701	* @returns VBox status code.
3702	* @param pVCpu Pointer to the VMCPU.
3703	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3704	* out-of-sync. Make sure to update the required fields
3705	* before using them.
3706	*
3707	* @remarks No-long-jump zone!!!
3708	*/
3709	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3710	{
3711	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3712	{
3713	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3714	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3715	#endif
3716	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3717	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3718	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3719	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3720	#else
3721	/* 64-bit host or hybrid host. */
3722	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3723	#endif
3724	}
3725	else
3726	{
3727	/* Guest is not in long mode, use the 32-bit handler. */
3728	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3729	}
3730	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3731	return VINF_SUCCESS;
3732	}
3733
3734
3735	/**
3736	* Wrapper for running the guest code in VT-x.
3737	*
3738	* @returns VBox strict status code.
3739	* @param pVM Pointer to the VM.
3740	* @param pVCpu Pointer to the VMCPU.
3741	* @param pCtx Pointer to the guest-CPU context.
3742	*
3743	* @remarks No-long-jump zone!!!
3744	*/
3745	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3746	{
3747	/*
3748	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3749	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3750	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3751	*/
3752	#ifdef VBOX_WITH_KERNEL_USING_XMM
3753	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3754	#else
3755	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3756	#endif
3757	}
3758
3759
3760	/**
3761	* Report world-switch error and dump some useful debug info.
3762	*
3763	* @param pVM Pointer to the VM.
3764	* @param pVCpu Pointer to the VMCPU.
3765	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3766	* @param pCtx Pointer to the guest-CPU context.
3767	* @param pVmxTransient Pointer to the VMX transient structure (only
3768	* exitReason updated).
3769	*/
3770	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3771	{
3772	Assert(pVM);
3773	Assert(pVCpu);
3774	Assert(pCtx);
3775	Assert(pVmxTransient);
3776	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3777
3778	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3779	switch (rcVMRun)
3780	{
3781	case VERR_VMX_INVALID_VMXON_PTR:
3782	AssertFailed();
3783	break;
3784	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3785	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3786	{
3787	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3788	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3789	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3790	AssertRC(rc);
3791
3792	#ifdef VBOX_STRICT
3793	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3794	pVmxTransient->uExitReason));
3795	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3796	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3797	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3798	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3799	else
3800	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3801
3802	/* VMX control bits. */
3803	uint32_t u32Val;
3804	uint64_t u64Val;
3805	RTHCUINTREG uHCReg;
3806	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3807	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3808	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3809	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3810	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3811	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3812	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3813	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3814	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3815	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3816	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3817	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3818	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3819	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3820	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3821	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3822	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3823	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3824	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3826	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3827	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3828	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3829	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3830	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3831	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3832	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3833	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3834	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3835	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3836	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3837	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3838	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3839	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3840	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3841	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3842	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3843	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3844	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3845	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3846	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3847	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3848
3849	/* Guest bits. */
3850	RTGCUINTREG uGCReg;
3851	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3852	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3853	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3854	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3855	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3856	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3857	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3858	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3859
3860	/* Host bits. */
3861	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3862	Log(("Host CR0 %#RHr\n", uHCReg));
3863	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3864	Log(("Host CR3 %#RHr\n", uHCReg));
3865	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3866	Log(("Host CR4 %#RHr\n", uHCReg));
3867
3868	RTGDTR HostGdtr;
3869	PCX86DESCHC pDesc;
3870	ASMGetGDTR(&HostGdtr);
3871	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3872	Log(("Host CS %#08x\n", u32Val));
3873	if (u32Val < HostGdtr.cbGdt)
3874	{
3875	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3876	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3877	}
3878
3879	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3880	Log(("Host DS %#08x\n", u32Val));
3881	if (u32Val < HostGdtr.cbGdt)
3882	{
3883	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3884	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3885	}
3886
3887	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3888	Log(("Host ES %#08x\n", u32Val));
3889	if (u32Val < HostGdtr.cbGdt)
3890	{
3891	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3892	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3893	}
3894
3895	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3896	Log(("Host FS %#08x\n", u32Val));
3897	if (u32Val < HostGdtr.cbGdt)
3898	{
3899	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3900	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3901	}
3902
3903	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3904	Log(("Host GS %#08x\n", u32Val));
3905	if (u32Val < HostGdtr.cbGdt)
3906	{
3907	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3908	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3909	}
3910
3911	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3912	Log(("Host SS %#08x\n", u32Val));
3913	if (u32Val < HostGdtr.cbGdt)
3914	{
3915	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3916	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3917	}
3918
3919	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3920	Log(("Host TR %#08x\n", u32Val));
3921	if (u32Val < HostGdtr.cbGdt)
3922	{
3923	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3924	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3925	}
3926
3927	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3928	Log(("Host TR Base %#RHv\n", uHCReg));
3929	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3930	Log(("Host GDTR Base %#RHv\n", uHCReg));
3931	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3932	Log(("Host IDTR Base %#RHv\n", uHCReg));
3933	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3934	Log(("Host SYSENTER CS %#08x\n", u32Val));
3935	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3936	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3937	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3938	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3939	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3940	Log(("Host RSP %#RHv\n", uHCReg));
3941	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3942	Log(("Host RIP %#RHv\n", uHCReg));
3943	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3944	if (HMVMX_IS_64BIT_HOST_MODE())
3945	{
3946	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3947	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3948	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3949	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3950	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3951	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3952	}
3953	# endif
3954	#endif /* VBOX_STRICT */
3955	break;
3956	}
3957
3958	default:
3959	/* Impossible */
3960	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3961	break;
3962	}
3963	NOREF(pVM);
3964	}
3965
3966
3967	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3968	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3969	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3970	#endif
3971
3972	#ifdef VBOX_STRICT
3973	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3974	{
3975	switch (idxField)
3976	{
3977	case VMX_VMCS_GUEST_RIP:
3978	case VMX_VMCS_GUEST_RSP:
3979	case VMX_VMCS_GUEST_RFLAGS:
3980	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3981	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3982	case VMX_VMCS_GUEST_CR0:
3983	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3984	case VMX_VMCS_GUEST_CR4:
3985	case VMX_VMCS_GUEST_DR7:
3986	case VMX_VMCS32_GUEST_SYSENTER_CS:
3987	case VMX_VMCS_GUEST_SYSENTER_EIP:
3988	case VMX_VMCS_GUEST_SYSENTER_ESP:
3989	case VMX_VMCS32_GUEST_GDTR_LIMIT:
3990	case VMX_VMCS_GUEST_GDTR_BASE:
3991	case VMX_VMCS32_GUEST_IDTR_LIMIT:
3992	case VMX_VMCS_GUEST_IDTR_BASE:
3993	case VMX_VMCS16_GUEST_FIELD_CS:
3994	case VMX_VMCS32_GUEST_CS_LIMIT:
3995	case VMX_VMCS_GUEST_CS_BASE:
3996	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
3997	case VMX_VMCS16_GUEST_FIELD_DS:
3998	case VMX_VMCS32_GUEST_DS_LIMIT:
3999	case VMX_VMCS_GUEST_DS_BASE:
4000	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4001	case VMX_VMCS16_GUEST_FIELD_ES:
4002	case VMX_VMCS32_GUEST_ES_LIMIT:
4003	case VMX_VMCS_GUEST_ES_BASE:
4004	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4005	case VMX_VMCS16_GUEST_FIELD_FS:
4006	case VMX_VMCS32_GUEST_FS_LIMIT:
4007	case VMX_VMCS_GUEST_FS_BASE:
4008	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4009	case VMX_VMCS16_GUEST_FIELD_GS:
4010	case VMX_VMCS32_GUEST_GS_LIMIT:
4011	case VMX_VMCS_GUEST_GS_BASE:
4012	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4013	case VMX_VMCS16_GUEST_FIELD_SS:
4014	case VMX_VMCS32_GUEST_SS_LIMIT:
4015	case VMX_VMCS_GUEST_SS_BASE:
4016	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4017	case VMX_VMCS16_GUEST_FIELD_LDTR:
4018	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4019	case VMX_VMCS_GUEST_LDTR_BASE:
4020	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4021	case VMX_VMCS16_GUEST_FIELD_TR:
4022	case VMX_VMCS32_GUEST_TR_LIMIT:
4023	case VMX_VMCS_GUEST_TR_BASE:
4024	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4025	case VMX_VMCS32_RO_EXIT_REASON:
4026	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4027	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4028	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4029	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4030	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4031	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4032	case VMX_VMCS32_RO_IDT_INFO:
4033	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4034	case VMX_VMCS_GUEST_CR3:
4035	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4036	return true;
4037	}
4038	return false;
4039	}
4040
4041	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4042	{
4043	switch (idxField)
4044	{
4045	case VMX_VMCS_GUEST_LDTR_BASE:
4046	case VMX_VMCS_GUEST_TR_BASE:
4047	case VMX_VMCS_GUEST_GDTR_BASE:
4048	case VMX_VMCS_GUEST_IDTR_BASE:
4049	case VMX_VMCS_GUEST_SYSENTER_EIP:
4050	case VMX_VMCS_GUEST_SYSENTER_ESP:
4051	case VMX_VMCS_GUEST_CR0:
4052	case VMX_VMCS_GUEST_CR4:
4053	case VMX_VMCS_GUEST_CR3:
4054	case VMX_VMCS_GUEST_DR7:
4055	case VMX_VMCS_GUEST_RIP:
4056	case VMX_VMCS_GUEST_RSP:
4057	case VMX_VMCS_GUEST_CS_BASE:
4058	case VMX_VMCS_GUEST_DS_BASE:
4059	case VMX_VMCS_GUEST_ES_BASE:
4060	case VMX_VMCS_GUEST_FS_BASE:
4061	case VMX_VMCS_GUEST_GS_BASE:
4062	case VMX_VMCS_GUEST_SS_BASE:
4063	return true;
4064	}
4065	return false;
4066	}
4067	#endif /* VBOX_STRICT */
4068
4069	/**
4070	* Executes the specified handler in 64-bit mode.
4071	*
4072	* @returns VBox status code.
4073	* @param pVM Pointer to the VM.
4074	* @param pVCpu Pointer to the VMCPU.
4075	* @param pCtx Pointer to the guest CPU context.
4076	* @param pfnHandler Pointer to the RC handler function.
4077	* @param cbParam Number of parameters.
4078	* @param paParam Array of 32-bit parameters.
4079	*/
4080	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4081	uint32_t *paParam)
4082	{
4083	int rc, rc2;
4084	PHMGLOBLCPUINFO pCpu;
4085	RTHCPHYS HCPhysCpuPage;
4086	RTHCUINTREG uOldEFlags;
4087
4088	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4089	Assert(pfnHandler);
4090	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4091	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4092
4093	#ifdef VBOX_STRICT
4094	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4095	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4096
4097	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4098	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4099	#endif
4100
4101	/* Disable interrupts. */
4102	uOldEFlags = ASMIntDisableFlags();
4103
4104	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4105	RTCPUID idHostCpu = RTMpCpuId();
4106	CPUMR0SetLApic(pVM, idHostCpu);
4107	#endif
4108
4109	pCpu = HMR0GetCurrentCpu();
4110	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4111
4112	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4113	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4114
4115	/* Leave VMX Root Mode. */
4116	VMXDisable();
4117
4118	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4119
4120	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4121	CPUMSetHyperEIP(pVCpu, pfnHandler);
4122	for (int i = (int)cbParam - 1; i >= 0; i--)
4123	CPUMPushHyper(pVCpu, paParam[i]);
4124
4125	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4126
4127	/* Call the switcher. */
4128	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4129	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4130
4131	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4132	/* Make sure the VMX instructions don't cause #UD faults. */
4133	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4134
4135	/* Re-enter VMX Root Mode */
4136	rc2 = VMXEnable(HCPhysCpuPage);
4137	if (RT_FAILURE(rc2))
4138	{
4139	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4140	ASMSetFlags(uOldEFlags);
4141	return rc2;
4142	}
4143
4144	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4145	AssertRC(rc2);
4146	Assert(!(ASMGetFlags() & X86_EFL_IF));
4147	ASMSetFlags(uOldEFlags);
4148	return rc;
4149	}
4150
4151
4152	/**
4153	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4154	* supporting 64-bit guests.
4155	*
4156	* @returns VBox status code.
4157	* @param fResume Whether to VMLAUNCH or VMRESUME.
4158	* @param pCtx Pointer to the guest-CPU context.
4159	* @param pCache Pointer to the VMCS cache.
4160	* @param pVM Pointer to the VM.
4161	* @param pVCpu Pointer to the VMCPU.
4162	*/
4163	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4164	{
4165	uint32_t aParam[6];
4166	PHMGLOBLCPUINFO pCpu = NULL;
4167	RTHCPHYS HCPhysCpuPage = 0;
4168	int rc = VERR_INTERNAL_ERROR_5;
4169
4170	pCpu = HMR0GetCurrentCpu();
4171	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4172
4173	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4174	pCache->uPos = 1;
4175	pCache->interPD = PGMGetInterPaeCR3(pVM);
4176	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4177	#endif
4178
4179	#ifdef VBOX_STRICT
4180	pCache->TestIn.HCPhysCpuPage = 0;
4181	pCache->TestIn.HCPhysVmcs = 0;
4182	pCache->TestIn.pCache = 0;
4183	pCache->TestOut.HCPhysVmcs = 0;
4184	pCache->TestOut.pCache = 0;
4185	pCache->TestOut.pCtx = 0;
4186	pCache->TestOut.eflags = 0;
4187	#endif
4188
4189	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4190	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4191	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4192	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4193	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4194	aParam[5] = 0;
4195
4196	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4197	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4198	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4199	#endif
4200	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4201
4202	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4203	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4204	Assert(pCtx->dr[4] == 10);
4205	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4206	#endif
4207
4208	#ifdef VBOX_STRICT
4209	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4210	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4211	pVCpu->hm.s.vmx.HCPhysVmcs));
4212	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4213	pCache->TestOut.HCPhysVmcs));
4214	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4215	pCache->TestOut.pCache));
4216	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4217	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4218	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4219	pCache->TestOut.pCtx));
4220	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4221	#endif
4222	return rc;
4223	}
4224
4225
4226	/**
4227	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4228	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4229	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4230	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4231	*
4232	* @returns VBox status code.
4233	* @param pVM Pointer to the VM.
4234	* @param pVCpu Pointer to the VMCPU.
4235	*/
4236	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4237	{
4238	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4239	{ \
4240	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4241	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4242	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4243	}
4244
4245	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4246	{ \
4247	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4248	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4249	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4251	}
4252
4253	AssertPtr(pVM);
4254	AssertPtr(pVCpu);
4255	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4256
4257	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4258	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4259	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4260	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4261	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4262	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4263	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4264	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4265	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4266
4267	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4268	#if 0
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4278	#endif
4279
4280	/* 32-bit guest-state fields. */
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4285	/* Unused 32-bit guest-state fields. */
4286	#if 0
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4290	#endif
4291
4292	/* Natural width guest-state fields. */
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4309	/* Unused natural width guest-state fields. */
4310	#if 0
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4313	#endif
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4316
4317	if (pVM->hm.s.fNestedPaging)
4318	{
4319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4321	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4322	}
4323	else
4324	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4325
4326	#undef VMXLOCAL_INIT_VMCS_SELREG
4327	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4328	return VINF_SUCCESS;
4329	}
4330
4331
4332	/**
4333	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4334	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4335	* darwin, running 64-bit guests).
4336	*
4337	* @returns VBox status code.
4338	* @param pVCpu Pointer to the VMCPU.
4339	* @param idxField The VMCS field encoding.
4340	* @param u64Val 16, 32 or 64 bits value.
4341	*/
4342	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4343	{
4344	int rc;
4345	switch (idxField)
4346	{
4347	/*
4348	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4349	*/
4350	/* 64-bit Control fields. */
4351	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4352	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4353	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4354	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4355	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4356	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4357	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4358	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4359	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4360	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4361	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4362	case VMX_VMCS64_CTRL_EPTP_FULL:
4363	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4364	/* 64-bit Read-only data fields. */
4365	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4366	/* 64-bit Guest-state fields. */
4367	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4368	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4369	case VMX_VMCS64_GUEST_PAT_FULL:
4370	case VMX_VMCS64_GUEST_EFER_FULL:
4371	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4372	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4373	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4374	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4375	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4376	/* 64-bit Host-state fields. */
4377	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4378	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4379	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4380	{
4381	rc = VMXWriteVmcs32(idxField, u64Val);
4382	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4383	break;
4384	}
4385
4386	/*
4387	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4388	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4389	*/
4390	/* Natural-width Guest-state fields. */
4391	case VMX_VMCS_GUEST_CR0:
4392	case VMX_VMCS_GUEST_CR3:
4393	case VMX_VMCS_GUEST_CR4:
4394	case VMX_VMCS_GUEST_ES_BASE:
4395	case VMX_VMCS_GUEST_CS_BASE:
4396	case VMX_VMCS_GUEST_SS_BASE:
4397	case VMX_VMCS_GUEST_DS_BASE:
4398	case VMX_VMCS_GUEST_FS_BASE:
4399	case VMX_VMCS_GUEST_GS_BASE:
4400	case VMX_VMCS_GUEST_LDTR_BASE:
4401	case VMX_VMCS_GUEST_TR_BASE:
4402	case VMX_VMCS_GUEST_GDTR_BASE:
4403	case VMX_VMCS_GUEST_IDTR_BASE:
4404	case VMX_VMCS_GUEST_DR7:
4405	case VMX_VMCS_GUEST_RSP:
4406	case VMX_VMCS_GUEST_RIP:
4407	case VMX_VMCS_GUEST_RFLAGS:
4408	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4409	case VMX_VMCS_GUEST_SYSENTER_ESP:
4410	case VMX_VMCS_GUEST_SYSENTER_EIP:
4411	{
4412	if ((u64Val >> 32ULL) == 0)
4413	{
4414	/* If this field is 64-bit, VT-x will zero out the top bits. */
4415	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4416	}
4417	else
4418	{
4419	/* Assert that only the 32->64 switcher case should ever come here. */
4420	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4421	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4422	}
4423	break;
4424	}
4425
4426	default:
4427	{
4428	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4429	rc = VERR_INVALID_PARAMETER;
4430	break;
4431	}
4432	}
4433	AssertRCReturn(rc, rc);
4434	return rc;
4435	}
4436
4437
4438	/**
4439	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4440	* hosts (except darwin) for 64-bit guests.
4441	*
4442	* @param pVCpu Pointer to the VMCPU.
4443	* @param idxField The VMCS field encoding.
4444	* @param u64Val 16, 32 or 64 bits value.
4445	*/
4446	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4447	{
4448	AssertPtr(pVCpu);
4449	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4450
4451	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4452	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4453
4454	/* Make sure there are no duplicates. */
4455	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4456	{
4457	if (pCache->Write.aField[i] == idxField)
4458	{
4459	pCache->Write.aFieldVal[i] = u64Val;
4460	return VINF_SUCCESS;
4461	}
4462	}
4463
4464	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4465	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4466	pCache->Write.cValidEntries++;
4467	return VINF_SUCCESS;
4468	}
4469
4470
4471	/**
4472	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4473	*
4474	* @param pVCpu Pointer to the VMCPU.
4475	* @param pCache Pointer to the VMCS cache.
4476	*
4477	* @remarks No-long-jump zone!!!
4478	*/
4479	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4480	{
4481	AssertPtr(pCache);
4482	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4483	{
4484	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4485	AssertRC(rc);
4486	}
4487	pCache->Write.cValidEntries = 0;
4488	}
4489
4490
4491	/**
4492	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4493	*
4494	* @param pVCpu Pointer to the VMCPU.
4495	* @param pCache Pointer to the VMCS cache.
4496	*
4497	* @remarks No-long-jump zone!!!
4498	*/
4499	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4500	{
4501	AssertPtr(pCache);
4502	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4503	{
4504	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4505	AssertRC(rc);
4506	}
4507	}
4508	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4509
4510
4511	/**
4512	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4513	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4514	* timer.
4515	*
4516	* @returns VBox status code.
4517	* @param pVCpu Pointer to the VMCPU.
4518	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4519	* out-of-sync. Make sure to update the required fields
4520	* before using them.
4521	* @remarks No-long-jump zone!!!
4522	*/
4523	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4524	{
4525	int rc = VERR_INTERNAL_ERROR_5;
4526	bool fOffsettedTsc = false;
4527	PVM pVM = pVCpu->CTX_SUFF(pVM);
4528	if (pVM->hm.s.vmx.fUsePreemptTimer)
4529	{
4530	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4531
4532	/* Make sure the returned values have sane upper and lower boundaries. */
4533	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4534	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4535	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4536	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4537
4538	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4539	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4540	}
4541	else
4542	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4543
4544	if (fOffsettedTsc)
4545	{
4546	uint64_t u64CurTSC = ASMReadTSC();
4547	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4548	{
4549	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4550	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4551
4552	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4553	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4554	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4555	}
4556	else
4557	{
4558	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4559	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4560	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4561	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4562	}
4563	}
4564	else
4565	{
4566	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4567	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4568	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4569	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4570	}
4571	}
4572
4573
4574	/**
4575	* Determines if an exception is a contributory exception. Contributory
4576	* exceptions are ones which can cause double-faults. Page-fault is
4577	* intentionally not included here as it's a conditional contributory exception.
4578	*
4579	* @returns true if the exception is contributory, false otherwise.
4580	* @param uVector The exception vector.
4581	*/
4582	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4583	{
4584	switch (uVector)
4585	{
4586	case X86_XCPT_GP:
4587	case X86_XCPT_SS:
4588	case X86_XCPT_NP:
4589	case X86_XCPT_TS:
4590	case X86_XCPT_DE:
4591	return true;
4592	default:
4593	break;
4594	}
4595	return false;
4596	}
4597
4598
4599	/**
4600	* Sets an event as a pending event to be injected into the guest.
4601	*
4602	* @param pVCpu Pointer to the VMCPU.
4603	* @param u32IntrInfo The VM-entry interruption-information field.
4604	* @param cbInstr The VM-entry instruction length in bytes (for software
4605	* interrupts, exceptions and privileged software
4606	* exceptions).
4607	* @param u32ErrCode The VM-entry exception error code.
4608	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4609	* page-fault.
4610	*/
4611	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4612	RTGCUINTPTR GCPtrFaultAddress)
4613	{
4614	Assert(!pVCpu->hm.s.Event.fPending);
4615	pVCpu->hm.s.Event.fPending = true;
4616	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4617	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4618	pVCpu->hm.s.Event.cbInstr = cbInstr;
4619	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4620	}
4621
4622
4623	/**
4624	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4625	*
4626	* @param pVCpu Pointer to the VMCPU.
4627	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4628	* out-of-sync. Make sure to update the required fields
4629	* before using them.
4630	*/
4631	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4632	{
4633	/* Inject the double-fault. */
4634	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4635	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4636	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4637	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4638	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4639	}
4640
4641
4642	/**
4643	* Handle a condition that occurred while delivering an event through the guest
4644	* IDT.
4645	*
4646	* @returns VBox status code (informational error codes included).
4647	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4648	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4649	* continue execution of the guest which will delivery the #DF.
4650	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4651	*
4652	* @param pVCpu Pointer to the VMCPU.
4653	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4654	* out-of-sync. Make sure to update the required fields
4655	* before using them.
4656	* @param pVmxTransient Pointer to the VMX transient structure.
4657	*
4658	* @remarks No-long-jump zone!!!
4659	*/
4660	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4661	{
4662	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4663	AssertRC(rc);
4664	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4665	{
4666	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4667	AssertRCReturn(rc, rc);
4668
4669	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4670	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4671	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4672
4673	typedef enum
4674	{
4675	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4676	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4677	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4678	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4679	} VMXREFLECTXCPT;
4680
4681	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4682	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4683	RTGCUINTPTR GCPtrFaultAddress = 0;
4684	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4685	{
4686	enmReflect = VMXREFLECTXCPT_XCPT;
4687	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4688	&& uExitVector == X86_XCPT_PF)
4689	{
4690	GCPtrFaultAddress = pMixedCtx->cr2;
4691	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4692	}
4693	else if ( uExitVector == X86_XCPT_PF
4694	&& uIdtVector == X86_XCPT_PF)
4695	{
4696	pVmxTransient->fVectoringPF = true;
4697	GCPtrFaultAddress = pMixedCtx->cr2;
4698	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4699	}
4700	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4701	&& hmR0VmxIsContributoryXcpt(uExitVector)
4702	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4703	\|\| uIdtVector == X86_XCPT_PF))
4704	{
4705	enmReflect = VMXREFLECTXCPT_DF;
4706	}
4707	else if (uIdtVector == X86_XCPT_DF)
4708	enmReflect = VMXREFLECTXCPT_TF;
4709	}
4710	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4711	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4712	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4713	{
4714	/*
4715	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4716	* (whatever they are) as they reoccur when restarting the instruction.
4717	*/
4718	enmReflect = VMXREFLECTXCPT_XCPT;
4719	}
4720
4721	switch (enmReflect)
4722	{
4723	case VMXREFLECTXCPT_XCPT:
4724	{
4725	uint32_t u32ErrCode = 0;
4726	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4727	{
4728	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4729	AssertRCReturn(rc, rc);
4730	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4731	}
4732	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4733	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4734	rc = VINF_SUCCESS;
4735	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4736	break;
4737	}
4738
4739	case VMXREFLECTXCPT_DF:
4740	{
4741	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4742	rc = VINF_VMX_DOUBLE_FAULT;
4743	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4744	uExitVector));
4745	break;
4746	}
4747
4748	case VMXREFLECTXCPT_TF:
4749	{
4750	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4751	rc = VINF_EM_RESET;
4752	break;
4753	}
4754
4755	default:
4756	Assert(rc == VINF_SUCCESS);
4757	break;
4758	}
4759	}
4760	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4761	return rc;
4762	}
4763
4764
4765	/**
4766	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4767	*
4768	* @returns VBox status code.
4769	* @param pVCpu Pointer to the VMCPU.
4770	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4771	* out-of-sync. Make sure to update the required fields
4772	* before using them.
4773	*
4774	* @remarks No-long-jump zone!!!
4775	*/
4776	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4777	{
4778	int rc = VINF_SUCCESS;
4779	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4780	{
4781	RTGCUINTREG uVal = 0;
4782	RTCCUINTREG uShadow = 0;
4783	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4784	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4785	AssertRCReturn(rc, rc);
4786	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4787	CPUMSetGuestCR0(pVCpu, uVal);
4788	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4789	}
4790	return rc;
4791	}
4792
4793
4794	/**
4795	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4796	*
4797	* @returns VBox status code.
4798	* @param pVCpu Pointer to the VMCPU.
4799	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4800	* out-of-sync. Make sure to update the required fields
4801	* before using them.
4802	*
4803	* @remarks No-long-jump zone!!!
4804	*/
4805	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4806	{
4807	int rc = VINF_SUCCESS;
4808	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4809	{
4810	RTGCUINTREG uVal = 0;
4811	RTCCUINTREG uShadow = 0;
4812	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4813	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4814	AssertRCReturn(rc, rc);
4815	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4816	CPUMSetGuestCR4(pVCpu, uVal);
4817	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4818	}
4819	return rc;
4820	}
4821
4822
4823	/**
4824	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4825	*
4826	* @returns VBox status code.
4827	* @param pVCpu Pointer to the VMCPU.
4828	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4829	* out-of-sync. Make sure to update the required fields
4830	* before using them.
4831	*
4832	* @remarks No-long-jump zone!!!
4833	*/
4834	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4835	{
4836	int rc = VINF_SUCCESS;
4837	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4838	{
4839	RTGCUINTREG uVal = 0;
4840	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4841	AssertRCReturn(rc, rc);
4842	pMixedCtx->rip = uVal;
4843	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4844	}
4845	return rc;
4846	}
4847
4848
4849	/**
4850	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4851	*
4852	* @returns VBox status code.
4853	* @param pVCpu Pointer to the VMCPU.
4854	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4855	* out-of-sync. Make sure to update the required fields
4856	* before using them.
4857	*
4858	* @remarks No-long-jump zone!!!
4859	*/
4860	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4861	{
4862	int rc = VINF_SUCCESS;
4863	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4864	{
4865	RTGCUINTREG uVal = 0;
4866	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4867	AssertRCReturn(rc, rc);
4868	pMixedCtx->rsp = uVal;
4869	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4870	}
4871	return rc;
4872	}
4873
4874
4875	/**
4876	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4877	*
4878	* @returns VBox status code.
4879	* @param pVCpu Pointer to the VMCPU.
4880	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4881	* out-of-sync. Make sure to update the required fields
4882	* before using them.
4883	*
4884	* @remarks No-long-jump zone!!!
4885	*/
4886	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4887	{
4888	int rc = VINF_SUCCESS;
4889	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4890	{
4891	RTGCUINTREG uVal = 0;
4892	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4893	AssertRCReturn(rc, rc);
4894	pMixedCtx->rflags.u64 = uVal;
4895
4896	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4897	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4898	{
4899	PVM pVM = pVCpu->CTX_SUFF(pVM);
4900	Assert(pVM->hm.s.vmx.pRealModeTSS);
4901	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4902	pMixedCtx->eflags.Bits.u1VM = 0;
4903	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4904	}
4905
4906	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4907	}
4908	return rc;
4909	}
4910
4911
4912	/**
4913	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4914	* guest-CPU context.
4915	*/
4916	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4917	{
4918	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4919	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4920	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4921	return rc;
4922	}
4923
4924
4925	/**
4926	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4927	* from the guest-state area in the VMCS.
4928	*
4929	* @param pVCpu Pointer to the VMCPU.
4930	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4931	* out-of-sync. Make sure to update the required fields
4932	* before using them.
4933	*
4934	* @remarks No-long-jump zone!!!
4935	*/
4936	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4937	{
4938	uint32_t uIntrState = 0;
4939	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4940	AssertRC(rc);
4941
4942	if (!uIntrState)
4943	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4944	else
4945	{
4946	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4947	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4948	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4949	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4950	AssertRC(rc);
4951	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4952	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4953	}
4954	}
4955
4956
4957	/**
4958	* Saves the guest's activity state.
4959	*
4960	* @returns VBox status code.
4961	* @param pVCpu Pointer to the VMCPU.
4962	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4963	* out-of-sync. Make sure to update the required fields
4964	* before using them.
4965	*
4966	* @remarks No-long-jump zone!!!
4967	*/
4968	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4969	{
4970	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4971	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4972	return VINF_SUCCESS;
4973	}
4974
4975
4976	/**
4977	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4978	* the current VMCS into the guest-CPU context.
4979	*
4980	* @returns VBox status code.
4981	* @param pVCpu Pointer to the VMCPU.
4982	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4983	* out-of-sync. Make sure to update the required fields
4984	* before using them.
4985	*
4986	* @remarks No-long-jump zone!!!
4987	*/
4988	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4989	{
4990	int rc = VINF_SUCCESS;
4991	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4992	{
4993	uint32_t u32Val = 0;
4994	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4995	pMixedCtx->SysEnter.cs = u32Val;
4996	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4997	}
4998
4999	RTGCUINTREG uGCVal = 0;
5000	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5001	{
5002	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5003	pMixedCtx->SysEnter.eip = uGCVal;
5004	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5005	}
5006	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5007	{
5008	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5009	pMixedCtx->SysEnter.esp = uGCVal;
5010	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5011	}
5012	return rc;
5013	}
5014
5015
5016	/**
5017	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5018	* context.
5019	*
5020	* @returns VBox status code.
5021	* @param pVCpu Pointer to the VMCPU.
5022	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5023	* out-of-sync. Make sure to update the required fields
5024	* before using them.
5025	*
5026	* @remarks No-long-jump zone!!!
5027	*/
5028	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5029	{
5030	RTGCUINTREG uVal = 0;
5031	int rc = VINF_SUCCESS;
5032	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5033	{
5034	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5035	pMixedCtx->fs.u64Base = uVal;
5036	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5037	}
5038	return rc;
5039	}
5040
5041
5042	/**
5043	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5044	* context.
5045	*
5046	* @returns VBox status code.
5047	* @param pVCpu Pointer to the VMCPU.
5048	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5049	* out-of-sync. Make sure to update the required fields
5050	* before using them.
5051	*
5052	* @remarks No-long-jump zone!!!
5053	*/
5054	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5055	{
5056	RTGCUINTREG uVal = 0;
5057	int rc = VINF_SUCCESS;
5058	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5059	{
5060	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5061	pMixedCtx->gs.u64Base = uVal;
5062	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5063	}
5064	return rc;
5065	}
5066
5067
5068	/**
5069	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5070	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5071	*
5072	* @returns VBox status code.
5073	* @param pVCpu Pointer to the VMCPU.
5074	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5075	* out-of-sync. Make sure to update the required fields
5076	* before using them.
5077	*
5078	* @remarks No-long-jump zone!!!
5079	*/
5080	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5081	{
5082	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5083	return VINF_SUCCESS;
5084
5085	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5086	{
5087	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5088	pMsr += i;
5089	switch (pMsr->u32IndexMSR)
5090	{
5091	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5092	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5093	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5094	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5095	#if 0
5096	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5097	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5098	#endif
5099	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5100	default:
5101	{
5102	AssertFailed();
5103	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5104	}
5105	}
5106	}
5107	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5108	return VINF_SUCCESS;
5109	}
5110
5111
5112	/**
5113	* Saves the guest control registers from the current VMCS into the guest-CPU
5114	* context.
5115	*
5116	* @returns VBox status code.
5117	* @param pVCpu Pointer to the VMCPU.
5118	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5119	* out-of-sync. Make sure to update the required fields
5120	* before using them.
5121	*
5122	* @remarks No-long-jump zone!!!
5123	*/
5124	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5125	{
5126	RTGCUINTREG uVal = 0;
5127	RTGCUINTREG uShadow = 0;
5128	int rc = VINF_SUCCESS;
5129
5130	/* Guest CR0. Guest FPU. */
5131	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5132
5133	/* Guest CR4. */
5134	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5135	AssertRCReturn(rc, rc);
5136
5137	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5138	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5139	{
5140	PVM pVM = pVCpu->CTX_SUFF(pVM);
5141	if ( pVM->hm.s.fNestedPaging
5142	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5143	{
5144	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5145	if (pMixedCtx->cr3 != uVal)
5146	{
5147	CPUMSetGuestCR3(pVCpu, uVal);
5148	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5149	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5150	}
5151
5152	/* We require EFER to check PAE mode. */
5153	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5154
5155	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5156	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5157	{
5158	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5159	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5160	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5161	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5162	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5163	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5164	}
5165	AssertRCReturn(rc, rc);
5166	}
5167
5168	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5169	}
5170	return rc;
5171	}
5172
5173
5174	/**
5175	* Reads a guest segment register from the current VMCS into the guest-CPU
5176	* context.
5177	*
5178	* @returns VBox status code.
5179	* @param pVCpu Pointer to the VMCPU.
5180	* @param idxSel Index of the selector in the VMCS.
5181	* @param idxLimit Index of the segment limit in the VMCS.
5182	* @param idxBase Index of the segment base in the VMCS.
5183	* @param idxAccess Index of the access rights of the segment in the VMCS.
5184	* @param pSelReg Pointer to the segment selector.
5185	*
5186	* @remarks No-long-jump zone!!!
5187	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5188	* as that takes care of whether to read from the VMCS cache or not.
5189	*/
5190	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5191	PCPUMSELREG pSelReg)
5192	{
5193	uint32_t u32Val = 0;
5194	int rc = VMXReadVmcs32(idxSel, &u32Val);
5195	pSelReg->Sel = (uint16_t)u32Val;
5196	pSelReg->ValidSel = (uint16_t)u32Val;
5197	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5198
5199	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5200	pSelReg->u32Limit = u32Val;
5201
5202	RTGCUINTREG uGCVal = 0;
5203	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5204	pSelReg->u64Base = uGCVal;
5205
5206	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5207	pSelReg->Attr.u = u32Val;
5208	AssertRCReturn(rc, rc);
5209
5210	/*
5211	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5212	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5213	*/
5214	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5215	{
5216	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5217	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5218	}
5219	return rc;
5220	}
5221
5222
5223	/**
5224	* Saves the guest segment registers from the current VMCS into the guest-CPU
5225	* context.
5226	*
5227	* @returns VBox status code.
5228	* @param pVCpu Pointer to the VMCPU.
5229	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5230	* out-of-sync. Make sure to update the required fields
5231	* before using them.
5232	*
5233	* @remarks No-long-jump zone!!!
5234	*/
5235	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5236	{
5237	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5238	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5239	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5240	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5241	#else
5242	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5243	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5244	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5245	#endif
5246
5247	int rc = VINF_SUCCESS;
5248
5249	/* Guest segment registers. */
5250	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5251	{
5252	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5253	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5254	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5255	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5256	rc \|= VMXLOCAL_READ_SEG(ES, es);
5257	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5258	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5259	AssertRCReturn(rc, rc);
5260
5261	/* Restore segment attributes for real-on-v86 mode hack. */
5262	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5263	{
5264	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5265	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5266	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5267	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5268	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5269	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5270	}
5271	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5272	}
5273
5274	/* Guest LDTR. */
5275	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5276	{
5277	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5278	AssertRCReturn(rc, rc);
5279	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5280	}
5281
5282	/* Guest GDTR. */
5283	RTGCUINTREG uGCVal = 0;
5284	uint32_t u32Val = 0;
5285	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5286	{
5287	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5288	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5289	pMixedCtx->gdtr.pGdt = uGCVal;
5290	pMixedCtx->gdtr.cbGdt = u32Val;
5291	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5292	}
5293
5294	/* Guest IDTR. */
5295	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5296	{
5297	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5298	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5299	pMixedCtx->idtr.pIdt = uGCVal;
5300	pMixedCtx->idtr.cbIdt = u32Val;
5301	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5302	}
5303
5304	/* Guest TR. */
5305	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5306	{
5307	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5308
5309	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5310	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5311	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5312	AssertRCReturn(rc, rc);
5313	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5314	}
5315	return rc;
5316	}
5317
5318
5319	/**
5320	* Saves the guest debug registers from the current VMCS into the guest-CPU
5321	* context.
5322	*
5323	* @returns VBox status code.
5324	* @param pVCpu Pointer to the VMCPU.
5325	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5326	* out-of-sync. Make sure to update the required fields
5327	* before using them.
5328	*
5329	* @remarks No-long-jump zone!!!
5330	*/
5331	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5332	{
5333	int rc = VINF_SUCCESS;
5334	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5335	{
5336	RTGCUINTREG uVal;
5337	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5338	pMixedCtx->dr[7] = uVal;
5339
5340	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5341	}
5342	return rc;
5343	}
5344
5345
5346	/**
5347	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5348	*
5349	* @returns VBox status code.
5350	* @param pVCpu Pointer to the VMCPU.
5351	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5352	* out-of-sync. Make sure to update the required fields
5353	* before using them.
5354	*
5355	* @remarks No-long-jump zone!!!
5356	*/
5357	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5358	{
5359	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5360	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5361	return VINF_SUCCESS;
5362	}
5363
5364
5365	/**
5366	* Saves the entire guest state from the currently active VMCS into the
5367	* guest-CPU context. This essentially VMREADs all guest-data.
5368	*
5369	* @returns VBox status code.
5370	* @param pVCpu Pointer to the VMCPU.
5371	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5372	* out-of-sync. Make sure to update the required fields
5373	* before using them.
5374	*/
5375	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5376	{
5377	Assert(pVCpu);
5378	Assert(pMixedCtx);
5379
5380	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5381	return VINF_SUCCESS;
5382
5383	VMMRZCallRing3Disable(pVCpu);
5384	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5385	LogFunc(("\n"));
5386
5387	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5389
5390	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5391	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5392
5393	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5395
5396	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5397	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5398
5399	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5400	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5401
5402	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5403	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5404
5405	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5406	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5407
5408	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5409	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5410
5411	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5412	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5413
5414	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5415	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5416
5417	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5418	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5419
5420	VMMRZCallRing3Enable(pVCpu);
5421	return rc;
5422	}
5423
5424
5425	/**
5426	* Check per-VM and per-VCPU force flag actions that require us to go back to
5427	* ring-3 for one reason or another.
5428	*
5429	* @returns VBox status code (information status code included).
5430	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5431	* ring-3.
5432	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5433	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5434	* interrupts)
5435	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5436	* all EMTs to be in ring-3.
5437	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5438	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5439	* to the EM loop.
5440	*
5441	* @param pVM Pointer to the VM.
5442	* @param pVCpu Pointer to the VMCPU.
5443	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5444	* out-of-sync. Make sure to update the required fields
5445	* before using them.
5446	*/
5447	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5448	{
5449	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5450
5451	int rc = VERR_INTERNAL_ERROR_5;
5452	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5453	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5454	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5455	{
5456	/* We need the control registers now, make sure the guest-CPU context is updated. */
5457	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5458	AssertRCReturn(rc, rc);
5459
5460	/* Pending HM CR3 sync. */
5461	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5462	{
5463	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5464	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5465	}
5466	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5467	{
5468	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5469	AssertRC(rc);
5470	}
5471
5472	/* Pending PGM C3 sync. */
5473	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5474	{
5475	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5476	if (rc != VINF_SUCCESS)
5477	{
5478	AssertRC(rc);
5479	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5480	return rc;
5481	}
5482	}
5483
5484	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5485	/* -XXX- what was that about single stepping? */
5486	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5487	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5488	{
5489	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5490	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5491	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5492	return rc;
5493	}
5494
5495	/* Pending VM request packets, such as hardware interrupts. */
5496	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5497	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5498	{
5499	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5500	return VINF_EM_PENDING_REQUEST;
5501	}
5502
5503	/* Pending PGM pool flushes. */
5504	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5505	{
5506	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5507	return VINF_PGM_POOL_FLUSH_PENDING;
5508	}
5509
5510	/* Pending DMA requests. */
5511	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5512	{
5513	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5514	return VINF_EM_RAW_TO_R3;
5515	}
5516	}
5517
5518	/* Paranoia. */
5519	Assert(rc != VERR_EM_INTERPRETER);
5520	return VINF_SUCCESS;
5521	}
5522
5523
5524	/**
5525	* Converts any TRPM trap into a pending VMX event. This is typically used when
5526	* entering from ring-3 (not longjmp returns).
5527	*
5528	* @param pVCpu Pointer to the VMCPU.
5529	* @param pCtx Pointer to the guest-CPU context.
5530	*/
5531	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5532	{
5533	if (!TRPMHasTrap(pVCpu))
5534	{
5535	Assert(!pVCpu->hm.s.Event.fPending);
5536	return;
5537	}
5538
5539	uint8_t uVector;
5540	TRPMEVENT enmTrpmEvent;
5541	RTGCUINT uErrCode;
5542	RTGCUINTPTR GCPtrFaultAddress;
5543	uint8_t cbInstr;
5544
5545	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5546	AssertRC(rc);
5547
5548	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5549	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5550	uint32_t u32ErrCode = uErrCode;
5551	if (enmTrpmEvent == TRPM_TRAP)
5552	{
5553	switch (uVector)
5554	{
5555	case X86_XCPT_BP:
5556	case X86_XCPT_OF:
5557	{
5558	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5559	break;
5560	}
5561
5562	case X86_XCPT_PF:
5563	pCtx->cr2 = GCPtrFaultAddress;
5564	/* no break */
5565	case X86_XCPT_DF:
5566	case X86_XCPT_TS:
5567	case X86_XCPT_NP:
5568	case X86_XCPT_SS:
5569	case X86_XCPT_GP:
5570	case X86_XCPT_AC:
5571	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5572	/* no break! */
5573	default:
5574	{
5575	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5576	break;
5577	}
5578	}
5579	}
5580	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5581	{
5582	if (uVector != X86_XCPT_NMI)
5583	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5584	else
5585	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5586	}
5587	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5588	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5589	else
5590	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5591
5592	rc = TRPMResetTrap(pVCpu);
5593	AssertRC(rc);
5594	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5595	u32IntrInfo, enmTrpmEvent, cbInstr, u32ErrCode, GCPtrFaultAddress));
5596	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5597	}
5598
5599
5600	/**
5601	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5602	* VT-x to execute any instruction.
5603	*
5604	* @param pvCpu Pointer to the VMCPU.
5605	*/
5606	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5607	{
5608	if (pVCpu->hm.s.Event.fPending)
5609	{
5610	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5611	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5612	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5613	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5614
5615	/* If a trap was already pending, we did something wrong! */
5616	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5617
5618	TRPMEVENT enmTrapType;
5619	switch (uVectorType)
5620	{
5621	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5622	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5623	enmTrapType = TRPM_HARDWARE_INT;
5624	break;
5625	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5626	enmTrapType = TRPM_SOFTWARE_INT;
5627	break;
5628	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5629	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5630	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5631	enmTrapType = TRPM_TRAP;
5632	break;
5633	default:
5634	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5635	enmTrapType = TRPM_32BIT_HACK;
5636	break;
5637	}
5638
5639	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5640	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5641	AssertRC(rc);
5642
5643	if (fErrorCodeValid)
5644	TRPMSetErrorCode(pVCpu, uErrorCode);
5645	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5646	&& uVector == X86_XCPT_PF)
5647	{
5648	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5649	}
5650	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5651	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5652	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5653	{
5654	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5655	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5656	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5657	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5658	}
5659	pVCpu->hm.s.Event.fPending = false;
5660	}
5661	}
5662
5663
5664	/**
5665	* Does the necessary state syncing before doing a longjmp to ring-3.
5666	*
5667	* @param pVM Pointer to the VM.
5668	* @param pVCpu Pointer to the VMCPU.
5669	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5670	* out-of-sync. Make sure to update the required fields
5671	* before using them.
5672	* @param rcExit The reason for exiting to ring-3. Can be
5673	* VINF_VMM_UNKNOWN_RING3_CALL.
5674	*
5675	* @remarks No-long-jmp zone!!!
5676	*/
5677	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5678	{
5679	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5680	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5681
5682	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5683	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5684	AssertRC(rc);
5685
5686	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5687	if (CPUMIsGuestFPUStateActive(pVCpu))
5688	{
5689	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5690	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5691	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5692	}
5693
5694	/* Restore debug registers if necessary and resync on next R0 reentry. */
5695	if (CPUMIsGuestDebugStateActive(pVCpu))
5696	{
5697	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5698	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5699	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5700	}
5701	else if (CPUMIsHyperDebugStateActive(pVCpu))
5702	{
5703	CPUMR0LoadHostDebugState(pVM, pVCpu);
5704	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5705	}
5706
5707	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5708	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5709	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5710	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5711	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5712	}
5713
5714
5715	/**
5716	* An action requires us to go back to ring-3. This function does the necessary
5717	* steps before we can safely return to ring-3. This is not the same as longjmps
5718	* to ring-3, this is voluntary.
5719	*
5720	* @param pVM Pointer to the VM.
5721	* @param pVCpu Pointer to the VMCPU.
5722	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5723	* out-of-sync. Make sure to update the required fields
5724	* before using them.
5725	* @param rcExit The reason for exiting to ring-3. Can be
5726	* VINF_VMM_UNKNOWN_RING3_CALL.
5727	*/
5728	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5729	{
5730	Assert(pVM);
5731	Assert(pVCpu);
5732	Assert(pMixedCtx);
5733	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5734
5735	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5736	{
5737	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5738	return;
5739	}
5740	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5741	{
5742	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5743	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5744	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5745	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5746	return;
5747	}
5748
5749	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5750	VMMRZCallRing3Disable(pVCpu);
5751	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5752
5753	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5754	hmR0VmxUpdateTRPM(pVCpu);
5755
5756	/* Sync. the guest state. */
5757	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5758	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5759
5760	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5761	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5762	\| CPUM_CHANGED_LDTR
5763	\| CPUM_CHANGED_GDTR
5764	\| CPUM_CHANGED_IDTR
5765	\| CPUM_CHANGED_TR
5766	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5767
5768	/* On our way back from ring-3 the following needs to be done. */
5769	/** @todo This can change with preemption hooks. */
5770	if (rcExit == VINF_EM_RAW_INTERRUPT)
5771	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5772	else
5773	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5774
5775	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5776	VMMRZCallRing3Enable(pVCpu);
5777	}
5778
5779
5780	/**
5781	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5782	* longjump to ring-3 and possibly get preempted.
5783	*
5784	* @param pVCpu Pointer to the VMCPU.
5785	* @param enmOperation The operation causing the ring-3 longjump.
5786	* @param pvUser The user argument (pointer to the possibly
5787	* out-of-date guest-CPU context).
5788	*
5789	* @remarks Must never be called with @a enmOperation ==
5790	* VMMCALLRING3_VM_R0_ASSERTION.
5791	*/
5792	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5793	{
5794	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5795	Assert(pVCpu);
5796	Assert(pvUser);
5797	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5798	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5799
5800	VMMRZCallRing3Disable(pVCpu);
5801	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5802	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5803	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5804	VMMRZCallRing3Enable(pVCpu);
5805	}
5806
5807
5808	/**
5809	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5810	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5811	*
5812	* @returns VBox status code.
5813	* @param pVCpu Pointer to the VMCPU.
5814	*/
5815	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5816	{
5817	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5818	{
5819	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5820	{
5821	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5822	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5823	AssertRC(rc);
5824	}
5825	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5826	}
5827
5828
5829	/**
5830	* Injects any pending events into the guest if the guest is in a state to
5831	* receive them.
5832	*
5833	* @returns VBox status code (informational status codes included).
5834	* @param pVCpu Pointer to the VMCPU.
5835	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5836	* out-of-sync. Make sure to update the required fields
5837	* before using them.
5838	*/
5839	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5840	{
5841	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5842	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5843	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5844	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5845
5846	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5847	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5848	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5849	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5850
5851	int rc = VINF_SUCCESS;
5852	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5853	{
5854	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5855	bool fInject = true;
5856	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5857	{
5858	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5859	AssertRCReturn(rc, rc);
5860	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5861	if ( fBlockInt
5862	\|\| fBlockSti
5863	\|\| fBlockMovSS)
5864	{
5865	fInject = false;
5866	}
5867	}
5868	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5869	&& ( fBlockMovSS
5870	\|\| fBlockSti))
5871	{
5872	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5873	fInject = false;
5874	}
5875
5876	if (fInject)
5877	{
5878	Log(("Injecting pending event\n"));
5879	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5880	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5881	AssertRCReturn(rc, rc);
5882	pVCpu->hm.s.Event.fPending = false;
5883	}
5884	else
5885	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5886	} /** @todo SMI. SMIs take priority over NMIs. */
5887	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5888	{
5889	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5890	if ( !fBlockMovSS
5891	&& !fBlockSti)
5892	{
5893	Log(("Injecting NMI\n"));
5894	RTGCUINTPTR uIntrInfo;
5895	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5896	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5897	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5898	AssertRCReturn(rc, rc);
5899	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5900	}
5901	else
5902	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5903	}
5904	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5905	{
5906	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5907	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5908	AssertRCReturn(rc, rc);
5909	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5910	if ( !fBlockInt
5911	&& !fBlockSti
5912	&& !fBlockMovSS)
5913	{
5914	uint8_t u8Interrupt;
5915	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5916	if (RT_SUCCESS(rc))
5917	{
5918	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5919	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5920	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5921	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5922	}
5923	else
5924	{
5925	/** @todo Does this actually happen? If not turn it into an assertion. */
5926	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5927	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5928	rc = VINF_SUCCESS;
5929	}
5930	}
5931	else
5932	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5933	}
5934
5935	/*
5936	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5937	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5938	*/
5939	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5940	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5941	int rc2 = VINF_SUCCESS;
5942	if ( fBlockSti
5943	\|\| fBlockMovSS)
5944	{
5945	if (!DBGFIsStepping(pVCpu))
5946	{
5947	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5948	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5949	{
5950	/*
5951	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5952	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5953	*/
5954	rc2 = VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5955	}
5956	}
5957	else
5958	{
5959	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5960	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5961	uIntrState = 0;
5962	}
5963	}
5964
5965	/*
5966	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5967	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5968	*/
5969	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5970	AssertRC(rc2);
5971
5972	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5973	return rc;
5974	}
5975
5976
5977	/**
5978	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5979	*
5980	* @param pVCpu Pointer to the VMCPU.
5981	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5982	* out-of-sync. Make sure to update the required fields
5983	* before using them.
5984	*/
5985	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5986	{
5987	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5988	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5989	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5990	}
5991
5992
5993	/**
5994	* Injects a double-fault (#DF) exception into the VM.
5995	*
5996	* @returns VBox status code (informational status code included).
5997	* @param pVCpu Pointer to the VMCPU.
5998	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5999	* out-of-sync. Make sure to update the required fields
6000	* before using them.
6001	*/
6002	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6003	{
6004	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6005	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6006	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6007	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6008	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6009	}
6010
6011
6012	/**
6013	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6014	*
6015	* @param pVCpu Pointer to the VMCPU.
6016	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6017	* out-of-sync. Make sure to update the required fields
6018	* before using them.
6019	*/
6020	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6021	{
6022	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6023	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6024	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6025	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6026	}
6027
6028
6029	/**
6030	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6031	*
6032	* @param pVCpu Pointer to the VMCPU.
6033	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6034	* out-of-sync. Make sure to update the required fields
6035	* before using them.
6036	* @param cbInstr The value of RIP that is to be pushed on the guest
6037	* stack.
6038	*/
6039	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6040	{
6041	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6042	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6043	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6044	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6045	}
6046
6047
6048	/**
6049	* Injects a general-protection (#GP) fault into the VM.
6050	*
6051	* @returns VBox status code (informational status code included).
6052	* @param pVCpu Pointer to the VMCPU.
6053	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6054	* out-of-sync. Make sure to update the required fields
6055	* before using them.
6056	* @param u32ErrorCode The error code associated with the #GP.
6057	*/
6058	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6059	uint32_t *puIntrState)
6060	{
6061	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6062	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6063	if (fErrorCodeValid)
6064	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6065	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6066	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6067	}
6068
6069
6070	/**
6071	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6072	*
6073	* @param pVCpu Pointer to the VMCPU.
6074	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6075	* out-of-sync. Make sure to update the required fields
6076	* before using them.
6077	* @param uVector The software interrupt vector number.
6078	* @param cbInstr The value of RIP that is to be pushed on the guest
6079	* stack.
6080	*/
6081	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6082	{
6083	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6084	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6085	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6086	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6087	}
6088
6089
6090	/**
6091	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6092	* stack.
6093	*
6094	* @returns VBox status code (information status code included).
6095	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6096	* @param pVM Pointer to the VM.
6097	* @param pMixedCtx Pointer to the guest-CPU context.
6098	* @param uValue The value to push to the guest stack.
6099	*/
6100	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6101	{
6102	/*
6103	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6104	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6105	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6106	*/
6107	if (pMixedCtx->sp == 1)
6108	return VINF_EM_RESET;
6109	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6110	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6111	AssertRCReturn(rc, rc);
6112	return rc;
6113	}
6114
6115
6116	/**
6117	* Injects an event into the guest upon VM-entry by updating the relevant fields
6118	* in the VM-entry area in the VMCS.
6119	*
6120	* @returns VBox status code (informational error codes included).
6121	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6122	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6123	*
6124	* @param pVCpu Pointer to the VMCPU.
6125	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6126	* out-of-sync. Make sure to update the required fields
6127	* before using them.
6128	* @param u64IntrInfo The VM-entry interruption-information field.
6129	* @param cbInstr The VM-entry instruction length in bytes (for software
6130	* interrupts, exceptions and privileged software
6131	* exceptions).
6132	* @param u32ErrCode The VM-entry exception error code.
6133	* @param puIntrState Pointer to the current guest interruptibility-state.
6134	* This interruptibility-state will be updated if
6135	* necessary. This cannot not be NULL.
6136	*
6137	* @remarks No-long-jump zone!!!
6138	* @remarks Requires CR0!
6139	*/
6140	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6141	uint32_t u32ErrCode, uint32_t *puIntrState)
6142	{
6143	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6144	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6145	Assert(puIntrState);
6146	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6147
6148	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6149	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6150
6151	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6152	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6153	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6154
6155	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6156
6157	/* We require CR0 to check if the guest is in real-mode. */
6158	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6159	AssertRCReturn(rc, rc);
6160
6161	/*
6162	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6163	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6164	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6165	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6166	*/
6167	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6168	{
6169	PVM pVM = pVCpu->CTX_SUFF(pVM);
6170	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6171	{
6172	Assert(PDMVmmDevHeapIsEnabled(pVM));
6173	Assert(pVM->hm.s.vmx.pRealModeTSS);
6174
6175	/* Save the required guest state bits from the VMCS. */
6176	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6177	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6178	AssertRCReturn(rc, rc);
6179
6180	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6181	const size_t cbIdtEntry = 4;
6182	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6183	{
6184	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6185	if (uVector == X86_XCPT_DF)
6186	return VINF_EM_RESET;
6187	else if (uVector == X86_XCPT_GP)
6188	{
6189	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6190	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6191	}
6192
6193	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6194	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6195	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6196	}
6197
6198	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6199	uint16_t uGuestIp = pMixedCtx->ip;
6200	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6201	{
6202	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6203	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6204	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6205	}
6206	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6207	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6208
6209	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6210	uint16_t offIdtEntry = 0;
6211	RTSEL selIdtEntry = 0;
6212	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6213	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6214	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6215	AssertRCReturn(rc, rc);
6216
6217	/* Construct the stack frame for the interrupt/exception handler. */
6218	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6219	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6220	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6221	AssertRCReturn(rc, rc);
6222
6223	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6224	if (rc == VINF_SUCCESS)
6225	{
6226	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6227	pMixedCtx->rip = offIdtEntry;
6228	pMixedCtx->cs.Sel = selIdtEntry;
6229	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6230	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6231	\| HM_CHANGED_GUEST_RIP
6232	\| HM_CHANGED_GUEST_RFLAGS
6233	\| HM_CHANGED_GUEST_RSP;
6234
6235	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6236	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6237	{
6238	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6239	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6240	Log(("Clearing inhibition due to STI.\n"));
6241	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6242	}
6243	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6244	}
6245	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6246	return rc;
6247	}
6248	else
6249	{
6250	/*
6251	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6252	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6253	*/
6254	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6255	}
6256	}
6257
6258	/* Validate. */
6259	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6260	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6261	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6262
6263	/* Inject. */
6264	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6265	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6266	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6267	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6268	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6269	AssertRCReturn(rc, rc);
6270	return rc;
6271	}
6272
6273
6274	/**
6275	* Enters the VT-x session.
6276	*
6277	* @returns VBox status code.
6278	* @param pVM Pointer to the VM.
6279	* @param pVCpu Pointer to the VMCPU.
6280	* @param pCpu Pointer to the CPU info struct.
6281	*/
6282	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6283	{
6284	AssertPtr(pVM);
6285	AssertPtr(pVCpu);
6286	Assert(pVM->hm.s.vmx.fSupported);
6287	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6288	NOREF(pCpu);
6289
6290	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6291
6292	/* Make sure we're in VMX root mode. */
6293	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6294	if (!(u32HostCR4 & X86_CR4_VMXE))
6295	{
6296	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6297	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6298	}
6299
6300	/* Load the active VMCS as the current one. */
6301	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6302	if (RT_FAILURE(rc))
6303	return rc;
6304
6305	/** @todo this will change with preemption hooks where can can VMRESUME as long
6306	* as we're no preempted. */
6307	pVCpu->hm.s.fResumeVM = false;
6308	return VINF_SUCCESS;
6309	}
6310
6311
6312	/**
6313	* Leaves the VT-x session.
6314	*
6315	* @returns VBox status code.
6316	* @param pVM Pointer to the VM.
6317	* @param pVCpu Pointer to the VMCPU.
6318	* @param pCtx Pointer to the guest-CPU context.
6319	*/
6320	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6321	{
6322	AssertPtr(pVCpu);
6323	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6324	NOREF(pVM);
6325	NOREF(pCtx);
6326
6327	/** @todo this will change with preemption hooks where we only VMCLEAR when
6328	* we are actually going to be preempted, not all the time like we
6329	* currently do. */
6330	/*
6331	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6332	* and mark the VMCS launch-state as "clear".
6333	*/
6334	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6335	return rc;
6336	}
6337
6338
6339	/**
6340	* Saves the host state in the VMCS host-state.
6341	* Sets up the VM-exit MSR-load area.
6342	*
6343	* The CPU state will be loaded from these fields on every successful VM-exit.
6344	*
6345	* @returns VBox status code.
6346	* @param pVM Pointer to the VM.
6347	* @param pVCpu Pointer to the VMCPU.
6348	*
6349	* @remarks No-long-jump zone!!!
6350	*/
6351	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6352	{
6353	AssertPtr(pVM);
6354	AssertPtr(pVCpu);
6355	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6356
6357	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6358
6359	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6360	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6361	return VINF_SUCCESS;
6362
6363	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6364	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6365
6366	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6367	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6368
6369	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6370	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6371
6372	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6373	return rc;
6374	}
6375
6376
6377	/**
6378	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6379	* loaded from these fields on every successful VM-entry.
6380	*
6381	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6382	* Sets up the VM-entry controls.
6383	* Sets up the appropriate VMX non-root function to execute guest code based on
6384	* the guest CPU mode.
6385	*
6386	* @returns VBox status code.
6387	* @param pVM Pointer to the VM.
6388	* @param pVCpu Pointer to the VMCPU.
6389	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6390	* out-of-sync. Make sure to update the required fields
6391	* before using them.
6392	*
6393	* @remarks No-long-jump zone!!!
6394	*/
6395	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6396	{
6397	AssertPtr(pVM);
6398	AssertPtr(pVCpu);
6399	AssertPtr(pMixedCtx);
6400	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6401
6402	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6403
6404	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6405
6406	/* Determine real-on-v86 mode. */
6407	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6408	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6409	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6410	{
6411	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6412	}
6413
6414	/** @todo if the order of loading is important, inform it via comments here */
6415	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6416	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6417
6418	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6419	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6420
6421	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6422	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6423
6424	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6425	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6426
6427	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6428	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6429
6430	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6431	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6432
6433	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6434	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6435
6436	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6437	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6438
6439	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6440	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6441
6442	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6443	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6444
6445	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6446	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6447	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6448
6449	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6450	return rc;
6451	}
6452
6453
6454	/**
6455	* Does the preparations before executing guest code in VT-x.
6456	*
6457	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6458	* recompiler. We must be cautious what we do here regarding committing
6459	* guest-state information into the the VMCS assuming we assuredly execute the
6460	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6461	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6462	* that the recompiler can (and should) use them when it resumes guest
6463	* execution. Otherwise such operations must be done when we can no longer
6464	* exit to ring-3.
6465	*
6466	* @returns VBox status code (informational status codes included).
6467	* @retval VINF_SUCCESS if we can proceed with running the guest.
6468	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6469	* into the guest.
6470	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6471	*
6472	* @param pVM Pointer to the VM.
6473	* @param pVCpu Pointer to the VMCPU.
6474	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6475	* out-of-sync. Make sure to update the required fields
6476	* before using them.
6477	* @param pVmxTransient Pointer to the VMX transient structure.
6478	*
6479	* @remarks Called with preemption disabled.
6480	*/
6481	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6482	{
6483	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6484
6485	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6486	PGMRZDynMapFlushAutoSet(pVCpu);
6487	#endif
6488
6489	/* Check force flag actions that might require us to go back to ring-3. */
6490	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6491	if (rc != VINF_SUCCESS)
6492	return rc;
6493
6494	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6495	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6496	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6497	{
6498	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6499	RTGCPHYS GCPhysApicBase;
6500	GCPhysApicBase = pMixedCtx->msrApicBase;
6501	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6502
6503	/* Unalias any existing mapping. */
6504	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6505	AssertRCReturn(rc, rc);
6506
6507	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6508	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6509	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6510	AssertRCReturn(rc, rc);
6511
6512	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6513	}
6514
6515	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6516	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6517	pVmxTransient->uEFlags = ASMIntDisableFlags();
6518	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6519	{
6520	ASMSetFlags(pVmxTransient->uEFlags);
6521	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6522	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6523	return VINF_EM_RAW_INTERRUPT;
6524	}
6525	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6526	#endif
6527
6528	/*
6529	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6530	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6531	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6532	*/
6533	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6534	AssertRCReturn(rc, rc);
6535	return rc;
6536	}
6537
6538
6539	/**
6540	* Prepares to run guest code in VT-x and we've committed to doing so. This
6541	* means there is no backing out to ring-3 or anywhere else at this
6542	* point.
6543	*
6544	* @param pVM Pointer to the VM.
6545	* @param pVCpu Pointer to the VMCPU.
6546	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6547	* out-of-sync. Make sure to update the required fields
6548	* before using them.
6549	* @param pVmxTransient Pointer to the VMX transient structure.
6550	*
6551	* @remarks Called with preemption disabled.
6552	* @remarks No-long-jump zone!!!
6553	*/
6554	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6555	{
6556	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6557	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6558
6559	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6560	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6561	pVmxTransient->uEFlags = ASMIntDisableFlags();
6562	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6563	#endif
6564
6565	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6566	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6567	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6568	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6569	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6570	#endif
6571	int rc = VINF_SUCCESS;
6572	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6573	{
6574	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6575	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6576	}
6577	else if (pVCpu->hm.s.fContextUseFlags)
6578	{
6579	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6580	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6581	}
6582	AssertRC(rc);
6583	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6584
6585	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6586	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6587	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6588
6589	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6590	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6591
6592	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6593	{
6594	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6595	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6596	}
6597
6598	/*
6599	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6600	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6601	*/
6602	if (pVM->hm.s.fTPRPatchingActive)
6603	{
6604	Assert(!CPUMIsGuestInLongMode(pVCpu));
6605
6606	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6607	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6608	AssertRC(rc);
6609
6610	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6611	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6612	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6613	see hmR0VmxLoadGuestApicState(). */
6614	}
6615
6616	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6617	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6618	to start executing. */
6619	}
6620
6621
6622	/**
6623	* Performs some essential restoration of state after running guest code in
6624	* VT-x.
6625	*
6626	* @param pVM Pointer to the VM.
6627	* @param pVCpu Pointer to the VMCPU.
6628	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6629	* out-of-sync. Make sure to update the required fields
6630	* before using them.
6631	* @param pVmxTransient Pointer to the VMX transient structure.
6632	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6633	*
6634	* @remarks Called with interrupts disabled.
6635	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6636	* unconditionally when it is safe to do so.
6637	*/
6638	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6639	{
6640	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6641	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6642
6643	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6644	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6645	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6646	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6647	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6648
6649	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6650	{
6651	/** @todo Find a way to fix hardcoding a guestimate. */
6652	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6653	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6654	}
6655
6656	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6657	Assert(!(ASMGetFlags() & X86_EFL_IF));
6658	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6659
6660	/* Restore the effects of TPR patching if any. */
6661	if (pVM->hm.s.fTPRPatchingActive)
6662	{
6663	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6664	AssertRC(rc);
6665	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6666	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6667	}
6668
6669	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6670	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6671
6672	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6673	uint32_t uExitReason;
6674	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6675	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6676	AssertRC(rc);
6677	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6678	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6679
6680	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6681	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6682
6683	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6684	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6685	{
6686	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6687	return;
6688	}
6689
6690	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6691	{
6692	/* Update the guest interruptibility-state from the VMCS. */
6693	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6694	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6695	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6696	AssertRC(rc);
6697	#endif
6698	/*
6699	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6700	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6701	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6702	*/
6703	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6704	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6705	{
6706	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6707	AssertRC(rc);
6708	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6709	}
6710	}
6711	}
6712
6713
6714	/**
6715	* Runs the guest code using VT-x.
6716	*
6717	* @returns VBox status code.
6718	* @param pVM Pointer to the VM.
6719	* @param pVCpu Pointer to the VMCPU.
6720	* @param pCtx Pointer to the guest-CPU context.
6721	*
6722	* @remarks Called with preemption disabled.
6723	*/
6724	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6725	{
6726	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6727	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6728
6729	VMXTRANSIENT VmxTransient;
6730	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6731	int rc = VERR_INTERNAL_ERROR_5;
6732	uint32_t cLoops = 0;
6733	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6734
6735	for (;; cLoops++)
6736	{
6737	Assert(!HMR0SuspendPending());
6738	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6739	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6740	(unsigned)RTMpCpuId(), cLoops));
6741
6742	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6743	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6744	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6745	if (rc != VINF_SUCCESS)
6746	break;
6747
6748	/*
6749	* No longjmps to ring-3 from this point on!!!
6750	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6751	* This also disables flushing of the R0-logger instance (if any).
6752	*/
6753	VMMRZCallRing3Disable(pVCpu);
6754	VMMRZCallRing3RemoveNotification(pVCpu);
6755	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6756
6757	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6758	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6759
6760	/*
6761	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6762	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6763	*/
6764	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6765	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6766	{
6767	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6768	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6769	return rc;
6770	}
6771
6772	/* Handle the VM-exit. */
6773	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6774	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6775	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6776	#ifdef HM_PROFILE_EXIT_DISPATCH
6777	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6778	#endif
6779	#ifdef HMVMX_USE_FUNCTION_TABLE
6780	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6781	#else
6782	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6783	#endif
6784	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6785	if (rc != VINF_SUCCESS)
6786	break;
6787	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6788	{
6789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6790	rc = VINF_EM_RAW_INTERRUPT;
6791	break;
6792	}
6793	}
6794
6795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6796	if (rc == VERR_EM_INTERPRETER)
6797	rc = VINF_EM_RAW_EMULATE_INSTR;
6798	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6799	return rc;
6800	}
6801
6802
6803	#ifndef HMVMX_USE_FUNCTION_TABLE
6804	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6805	{
6806	int rc;
6807	switch (rcReason)
6808	{
6809	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6811	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6812	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6813	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6814	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6815	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6816	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6817	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6818	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6819	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6820	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6821	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6822	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6823	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6824	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6825	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6826	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6827	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6828	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6829	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6842
6843	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6852
6853	case VMX_EXIT_VMCALL:
6854	case VMX_EXIT_VMCLEAR:
6855	case VMX_EXIT_VMLAUNCH:
6856	case VMX_EXIT_VMPTRLD:
6857	case VMX_EXIT_VMPTRST:
6858	case VMX_EXIT_VMREAD:
6859	case VMX_EXIT_VMRESUME:
6860	case VMX_EXIT_VMWRITE:
6861	case VMX_EXIT_VMXOFF:
6862	case VMX_EXIT_VMXON:
6863	case VMX_EXIT_INVEPT:
6864	case VMX_EXIT_INVVPID:
6865	case VMX_EXIT_VMFUNC:
6866	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6867	break;
6868	default:
6869	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6870	break;
6871	}
6872	return rc;
6873	}
6874	#endif
6875
6876
6877	/** Profiling macro. */
6878	#ifdef HM_PROFILE_EXIT_DISPATCH
6879	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6880	#else
6881	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6882	#endif
6883
6884
6885	#ifdef DEBUG
6886	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6887	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6888	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6889	# define VMX_ASSERT_PREEMPT_CPUID() \
6890	do \
6891	{ \
6892	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6893	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6894	} while (0)
6895
6896	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6897	do { \
6898	AssertPtr(pVCpu); \
6899	AssertPtr(pMixedCtx); \
6900	AssertPtr(pVmxTransient); \
6901	Assert(pVmxTransient->fVMEntryFailed == false); \
6902	Assert(ASMIntAreEnabled()); \
6903	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6904	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6905	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6906	(unsigned)pVCpu->idCpu)); \
6907	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6908	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6909	VMX_ASSERT_PREEMPT_CPUID(); \
6910	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6911	} while (0)
6912	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6913	do { \
6914	LogFunc(("\n")); \
6915	} while(0)
6916	#else /* Release builds */
6917	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6918	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6919	#endif
6920
6921
6922	/**
6923	* Advances the guest RIP after reading it from the VMCS.
6924	*
6925	* @returns VBox status code.
6926	* @param pVCpu Pointer to the VMCPU.
6927	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6928	* out-of-sync. Make sure to update the required fields
6929	* before using them.
6930	* @param pVmxTransient Pointer to the VMX transient structure.
6931	*
6932	* @remarks No-long-jump zone!!!
6933	*/
6934	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6935	{
6936	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6937	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6938	AssertRCReturn(rc, rc);
6939
6940	pMixedCtx->rip += pVmxTransient->cbInstr;
6941	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6942	return rc;
6943	}
6944
6945
6946	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6947	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6948	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6949	/**
6950	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6951	*/
6952	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6953	{
6954	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6955	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6956	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6957	Assert(ASMIntAreEnabled());
6958	return VINF_SUCCESS;
6959	#else
6960	return VINF_EM_RAW_INTERRUPT;
6961	#endif
6962	}
6963
6964
6965	/**
6966	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6967	*/
6968	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6969	{
6970	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6971	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6972
6973	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6974	AssertRCReturn(rc, rc);
6975
6976	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6977	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6978	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6979
6980	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6981	{
6982	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6983	return VINF_EM_RAW_INTERRUPT;
6984	}
6985
6986	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6987	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6988	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6989	{
6990	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6991	return VINF_SUCCESS;
6992	}
6993	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6994	{
6995	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6996	return rc;
6997	}
6998
6999	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7000	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7001	switch (uIntrType)
7002	{
7003	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7004	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7005	/* no break */
7006	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7007	{
7008	switch (uVector)
7009	{
7010	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7011	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7012	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7013	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7014	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7015	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7016	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7017	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7018	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7019	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7020	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7021	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7022	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7023	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7024	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7025	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7026	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7027	#endif
7028	default:
7029	{
7030	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7031	AssertRCReturn(rc, rc);
7032
7033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7034	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7035	{
7036	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7037	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7038	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7039	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7040	AssertRCReturn(rc, rc);
7041	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7042	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7043	0 /* GCPtrFaultAddress */);
7044	AssertRCReturn(rc, rc);
7045	}
7046	else
7047	{
7048	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7049	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7050	}
7051	break;
7052	}
7053	}
7054	break;
7055	}
7056
7057	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7058	default:
7059	{
7060	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7061	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7062	break;
7063	}
7064	}
7065	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7066	return rc;
7067	}
7068
7069
7070	/**
7071	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7072	*/
7073	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7074	{
7075	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7076
7077	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7078	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7079	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7080	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7081	AssertRCReturn(rc, rc);
7082
7083	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7084	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7085	return VINF_SUCCESS;
7086	}
7087
7088
7089	/**
7090	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7091	*/
7092	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7093	{
7094	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7095	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7096	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7097	}
7098
7099
7100	/**
7101	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7102	*/
7103	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7104	{
7105	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7107	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7108	}
7109
7110
7111	/**
7112	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7113	*/
7114	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7115	{
7116	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7117	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7118	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7119	}
7120
7121
7122	/**
7123	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7124	*/
7125	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7126	{
7127	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7128	PVM pVM = pVCpu->CTX_SUFF(pVM);
7129	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7130	if (RT_LIKELY(rc == VINF_SUCCESS))
7131	{
7132	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7133	Assert(pVmxTransient->cbInstr == 2);
7134	}
7135	else
7136	{
7137	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7138	rc = VERR_EM_INTERPRETER;
7139	}
7140	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7141	return rc;
7142	}
7143
7144
7145	/**
7146	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7147	*/
7148	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7149	{
7150	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7151	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7152	AssertRCReturn(rc, rc);
7153
7154	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7155	return VINF_EM_RAW_EMULATE_INSTR;
7156
7157	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7158	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7159	}
7160
7161
7162	/**
7163	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7164	*/
7165	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7166	{
7167	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7168	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7169	AssertRCReturn(rc, rc);
7170
7171	PVM pVM = pVCpu->CTX_SUFF(pVM);
7172	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7173	if (RT_LIKELY(rc == VINF_SUCCESS))
7174	{
7175	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7176	Assert(pVmxTransient->cbInstr == 2);
7177	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7178	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7179	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7180	}
7181	else
7182	{
7183	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7184	rc = VERR_EM_INTERPRETER;
7185	}
7186	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7187	return rc;
7188	}
7189
7190
7191	/**
7192	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7193	*/
7194	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7195	{
7196	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7197	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7198	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7199	AssertRCReturn(rc, rc);
7200
7201	PVM pVM = pVCpu->CTX_SUFF(pVM);
7202	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7203	if (RT_LIKELY(rc == VINF_SUCCESS))
7204	{
7205	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7206	Assert(pVmxTransient->cbInstr == 3);
7207	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7208	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7209	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7210	}
7211	else
7212	{
7213	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7214	rc = VERR_EM_INTERPRETER;
7215	}
7216	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7217	return rc;
7218	}
7219
7220
7221	/**
7222	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7223	*/
7224	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7225	{
7226	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7227	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7228	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7229	AssertRCReturn(rc, rc);
7230
7231	PVM pVM = pVCpu->CTX_SUFF(pVM);
7232	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7233	if (RT_LIKELY(rc == VINF_SUCCESS))
7234	{
7235	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7236	Assert(pVmxTransient->cbInstr == 2);
7237	}
7238	else
7239	{
7240	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7241	rc = VERR_EM_INTERPRETER;
7242	}
7243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7244	return rc;
7245	}
7246
7247
7248	/**
7249	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7250	*/
7251	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7252	{
7253	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7254	PVM pVM = pVCpu->CTX_SUFF(pVM);
7255	Assert(!pVM->hm.s.fNestedPaging);
7256
7257	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7258	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7259	AssertRCReturn(rc, rc);
7260
7261	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7262	rc = VBOXSTRICTRC_VAL(rc2);
7263	if (RT_LIKELY(rc == VINF_SUCCESS))
7264	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7265	else
7266	{
7267	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7268	pVmxTransient->uExitQualification, rc));
7269	}
7270	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7271	return rc;
7272	}
7273
7274
7275	/**
7276	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7277	*/
7278	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7279	{
7280	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7281	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7282	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7283	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7284	AssertRCReturn(rc, rc);
7285
7286	PVM pVM = pVCpu->CTX_SUFF(pVM);
7287	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7288	if (RT_LIKELY(rc == VINF_SUCCESS))
7289	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7290	else
7291	{
7292	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7293	rc = VERR_EM_INTERPRETER;
7294	}
7295	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7296	return rc;
7297	}
7298
7299
7300	/**
7301	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7302	*/
7303	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7304	{
7305	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7306	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7307	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7308	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7309	AssertRCReturn(rc, rc);
7310
7311	PVM pVM = pVCpu->CTX_SUFF(pVM);
7312	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7313	rc = VBOXSTRICTRC_VAL(rc2);
7314	if (RT_LIKELY( rc == VINF_SUCCESS
7315	\|\| rc == VINF_EM_HALT))
7316	{
7317	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7318	AssertRCReturn(rc3, rc3);
7319
7320	if ( rc == VINF_EM_HALT
7321	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7322	{
7323	rc = VINF_SUCCESS;
7324	}
7325	}
7326	else
7327	{
7328	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7329	rc = VERR_EM_INTERPRETER;
7330	}
7331	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7332	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7333	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7334	return rc;
7335	}
7336
7337
7338	/**
7339	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7340	*/
7341	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7342	{
7343	/*
7344	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7345	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7346	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7347	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7348	*/
7349	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7350	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7351	}
7352
7353
7354	/**
7355	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7356	*/
7357	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7358	{
7359	/*
7360	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7361	* root operation. If we get there there is something funny going on.
7362	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7363	*/
7364	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7365	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7366	}
7367
7368
7369	/**
7370	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7371	*/
7372	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7373	{
7374	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7375	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7376	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7377	}
7378
7379
7380	/**
7381	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7382	*/
7383	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7384	{
7385	/*
7386	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7387	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7388	* See Intel spec. 25.3 "Other Causes of VM-exits".
7389	*/
7390	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7391	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7397	* VM-exit.
7398	*/
7399	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7400	{
7401	/*
7402	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7403	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7404	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7405	*/
7406	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7407	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7408	}
7409
7410
7411	/**
7412	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7413	* VM-exit.
7414	*/
7415	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7416	{
7417	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7418	return VINF_EM_RESET;
7419	}
7420
7421
7422	/**
7423	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7424	*/
7425	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7426	{
7427	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7428	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7429	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7430	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7431	AssertRCReturn(rc, rc);
7432
7433	pMixedCtx->rip++;
7434	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7435	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7436	rc = VINF_SUCCESS;
7437	else
7438	rc = VINF_EM_HALT;
7439
7440	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7441	return rc;
7442	}
7443
7444
7445	/**
7446	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7447	*/
7448	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7449	{
7450	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7451	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7452	return VINF_SUCCESS;
7453	}
7454
7455
7456	/**
7457	* VM-exit handler for expiry of the VMX preemption timer.
7458	*/
7459	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7460	{
7461	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7462
7463	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7464	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7465	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7466
7467	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7468	PVM pVM = pVCpu->CTX_SUFF(pVM);
7469	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7470	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7471	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7472	}
7473
7474
7475	/**
7476	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7477	*/
7478	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7479	{
7480	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7481	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7482	/** @todo check if XSETBV is supported by the recompiler. */
7483	return VERR_EM_INTERPRETER;
7484	}
7485
7486
7487	/**
7488	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7489	*/
7490	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7491	{
7492	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7493	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7494	/** @todo implement EMInterpretInvpcid() */
7495	return VERR_EM_INTERPRETER;
7496	}
7497
7498
7499	/**
7500	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7501	* Error VM-exit.
7502	*/
7503	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7504	{
7505	uint32_t uIntrState;
7506	RTHCUINTREG uHCReg;
7507	uint64_t u64Val;
7508
7509	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7510	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7511	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7512	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7513	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7514	AssertRCReturn(rc, rc);
7515
7516	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7517	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7518	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7519	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7520
7521	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7522	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7523	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7524	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7525	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7526	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7527	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7528	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7529	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7530	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7531	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7532	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7533
7534	PVM pVM = pVCpu->CTX_SUFF(pVM);
7535	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7536
7537	return VERR_VMX_INVALID_GUEST_STATE;
7538	}
7539
7540
7541	/**
7542	* VM-exit handler for VM-entry failure due to an MSR-load
7543	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7544	*/
7545	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7546	{
7547	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7548	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7549	}
7550
7551
7552	/**
7553	* VM-exit handler for VM-entry failure due to a machine-check event
7554	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7555	*/
7556	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7557	{
7558	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7559	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7560	}
7561
7562
7563	/**
7564	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7565	* theory.
7566	*/
7567	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7568	{
7569	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7570	return VERR_VMX_UNDEFINED_EXIT_CODE;
7571	}
7572
7573
7574	/**
7575	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7576	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7577	* Conditional VM-exit.
7578	*/
7579	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7580	{
7581	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7582	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7584	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7585	return VERR_EM_INTERPRETER;
7586	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7587	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7588	}
7589
7590
7591	/**
7592	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7593	*/
7594	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7595	{
7596	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7597	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7598	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7599	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7600	return VERR_EM_INTERPRETER;
7601	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7602	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7603	}
7604
7605
7606	/**
7607	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7608	*/
7609	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7610	{
7611	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7612	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7613	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7614	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7615	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7616	AssertRCReturn(rc, rc);
7617
7618	PVM pVM = pVCpu->CTX_SUFF(pVM);
7619	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7620	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7621	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7622	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7623
7624	if (RT_LIKELY(rc == VINF_SUCCESS))
7625	{
7626	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7627	Assert(pVmxTransient->cbInstr == 2);
7628	}
7629	return rc;
7630	}
7631
7632
7633	/**
7634	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7635	*/
7636	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7637	{
7638	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7639	PVM pVM = pVCpu->CTX_SUFF(pVM);
7640	int rc = VINF_SUCCESS;
7641
7642	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7643	if ( pVM->hm.s.fTPRPatchingActive
7644	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7645	{
7646	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7647	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7648	{
7649	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7650	AssertRC(rc);
7651	}
7652
7653	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7654	Assert(pVmxTransient->cbInstr == 2);
7655	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7656	return VINF_SUCCESS;
7657	}
7658
7659	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7660	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7661	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7662	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7663	AssertRCReturn(rc, rc);
7664	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7665
7666	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7667	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7669
7670	if (RT_LIKELY(rc == VINF_SUCCESS))
7671	{
7672	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7673
7674	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7675	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7676	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7677	{
7678	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7679	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7680	}
7681	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7682	{
7683	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7684	AssertRCReturn(rc, rc);
7685	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7686	}
7687
7688	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7689	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7690	{
7691	switch (pMixedCtx->ecx)
7692	{
7693	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7694	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7695	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7696	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7697	case MSR_K8_FS_BASE: /* no break */
7698	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7699	}
7700	}
7701	#ifdef VBOX_STRICT
7702	else
7703	{
7704	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7705	switch (pMixedCtx->ecx)
7706	{
7707	case MSR_IA32_SYSENTER_CS:
7708	case MSR_IA32_SYSENTER_EIP:
7709	case MSR_IA32_SYSENTER_ESP:
7710	case MSR_K8_FS_BASE:
7711	case MSR_K8_GS_BASE:
7712	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7713	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7714
7715	case MSR_K8_LSTAR:
7716	case MSR_K6_STAR:
7717	case MSR_K8_SF_MASK:
7718	case MSR_K8_TSC_AUX:
7719	{
7720	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7721	pMixedCtx->ecx));
7722	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7723	}
7724
7725	case MSR_K8_KERNEL_GS_BASE:
7726	{
7727	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7728	pMixedCtx->ecx));
7729	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7730	}
7731	}
7732	}
7733	#endif /* VBOX_STRICT */
7734	}
7735	return rc;
7736	}
7737
7738
7739	/**
7740	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7741	*/
7742	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7743	{
7744	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7745	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7747	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7748	return VERR_EM_INTERPRETER;
7749	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7750	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7751	}
7752
7753
7754	/**
7755	* VM-exit handler for when the TPR value is lowered below the specified
7756	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7757	*/
7758	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7759	{
7760	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7761	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7762
7763	/*
7764	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7765	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7766	* resume guest execution.
7767	*/
7768	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7770	return VINF_SUCCESS;
7771	}
7772
7773
7774	/**
7775	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7776	* VM-exit.
7777	*
7778	* @retval VINF_SUCCESS when guest execution can continue.
7779	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7780	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7781	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7782	* recompiler.
7783	*/
7784	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7785	{
7786	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7787	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7788	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7789	AssertRCReturn(rc, rc);
7790
7791	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7792	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7793	PVM pVM = pVCpu->CTX_SUFF(pVM);
7794	switch (uAccessType)
7795	{
7796	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7797	{
7798	#if 0
7799	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7800	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7801	#else
7802	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7803	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7804	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7805	#endif
7806	AssertRCReturn(rc, rc);
7807
7808	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7809	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7810	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7811	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7812
7813	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7814	{
7815	case 0: /* CR0 */
7816	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7817	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7818	break;
7819	case 2: /* C2 **/
7820	/* Nothing to do here, CR2 it's not part of the VMCS. */
7821	break;
7822	case 3: /* CR3 */
7823	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7824	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7825	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7826	break;
7827	case 4: /* CR4 */
7828	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7829	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7830	break;
7831	case 8: /* CR8 */
7832	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7833	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7834	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7835	break;
7836	default:
7837	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7838	break;
7839	}
7840
7841	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7842	break;
7843	}
7844
7845	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7846	{
7847	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7848	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7849	AssertRCReturn(rc, rc);
7850	Assert( !pVM->hm.s.fNestedPaging
7851	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7852	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7853
7854	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7855	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7856	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7857
7858	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7859	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7860	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7861	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7863	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7864	break;
7865	}
7866
7867	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7868	{
7869	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7870	AssertRCReturn(rc, rc);
7871	rc = EMInterpretCLTS(pVM, pVCpu);
7872	AssertRCReturn(rc, rc);
7873	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7874	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7875	Log(("CRX CLTS write rc=%d\n", rc));
7876	break;
7877	}
7878
7879	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7880	{
7881	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7882	AssertRCReturn(rc, rc);
7883	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7884	if (RT_LIKELY(rc == VINF_SUCCESS))
7885	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7886	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7887	Log(("CRX LMSW write rc=%d\n", rc));
7888	break;
7889	}
7890
7891	default:
7892	{
7893	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7894	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7895	}
7896	}
7897
7898	/* Validate possible error codes. */
7899	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7900	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7901	if (RT_SUCCESS(rc))
7902	{
7903	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7904	AssertRCReturn(rc2, rc2);
7905	}
7906
7907	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7908	return rc;
7909	}
7910
7911
7912	/**
7913	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7914	* VM-exit.
7915	*/
7916	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7917	{
7918	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7919	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7920
7921	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7922	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7923	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7924	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7925	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7926	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7927	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7928	AssertRCReturn(rc, rc);
7929	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7930
7931	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7932	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7933	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7934	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7935	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7936	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7937	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7938
7939	/* I/O operation lookup arrays. */
7940	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7941	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7942
7943	const uint32_t cbSize = s_aIOSize[uIOWidth];
7944	const uint32_t cbInstr = pVmxTransient->cbInstr;
7945	PVM pVM = pVCpu->CTX_SUFF(pVM);
7946	if (fIOString)
7947	{
7948	/* INS/OUTS - I/O String instruction. */
7949	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7950	/** @todo for now manually disassemble later optimize by getting the fields from
7951	* the VMCS. */
7952	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7953	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7954	* segment prefix info. */
7955	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7956	if (RT_SUCCESS(rc))
7957	{
7958	if (fIOWrite)
7959	{
7960	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7961	(DISCPUMODE)pDis->uAddrMode, cbSize);
7962	rc = VBOXSTRICTRC_VAL(rc2);
7963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7964	}
7965	else
7966	{
7967	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7968	(DISCPUMODE)pDis->uAddrMode, cbSize);
7969	rc = VBOXSTRICTRC_VAL(rc2);
7970	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7971	}
7972	}
7973	else
7974	{
7975	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7976	rc = VINF_EM_RAW_EMULATE_INSTR;
7977	}
7978	}
7979	else
7980	{
7981	/* IN/OUT - I/O instruction. */
7982	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7983	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7984	if (fIOWrite)
7985	{
7986	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7987	rc = VBOXSTRICTRC_VAL(rc2);
7988	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7989	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7990	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7991	}
7992	else
7993	{
7994	uint32_t u32Result = 0;
7995	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7996	rc = VBOXSTRICTRC_VAL(rc2);
7997	if (IOM_SUCCESS(rc))
7998	{
7999	/* Save result of I/O IN instr. in AL/AX/EAX. */
8000	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8001	}
8002	else if (rc == VINF_IOM_R3_IOPORT_READ)
8003	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8005	}
8006	}
8007
8008	if (IOM_SUCCESS(rc))
8009	{
8010	pMixedCtx->rip += cbInstr;
8011	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8012	if (RT_LIKELY(rc == VINF_SUCCESS))
8013	{
8014	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8015	AssertRCReturn(rc, rc);
8016
8017	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8018	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8019	{
8020	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8021	for (unsigned i = 0; i < 4; i++)
8022	{
8023	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8024	if ( ( uIOPort >= pMixedCtx->dr[i]
8025	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8026	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8027	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8028	{
8029	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8030	uint64_t uDR6 = ASMGetDR6();
8031
8032	/* Clear all breakpoint status flags and set the one we just hit. */
8033	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8034	uDR6 \|= (uint64_t)RT_BIT(i);
8035
8036	/*
8037	* Note: AMD64 Architecture Programmer's Manual 13.1:
8038	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8039	* be cleared by software after the contents have been read.
8040	*/
8041	ASMSetDR6(uDR6);
8042
8043	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8044	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8045
8046	/* Paranoia. */
8047	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8048	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8049	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8050
8051	/* Resync DR7 */
8052	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8053	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8054
8055	/* Set #DB to be injected into the VM and continue guest execution. */
8056	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8057	break;
8058	}
8059	}
8060	}
8061	}
8062	}
8063
8064	#ifdef DEBUG
8065	if (rc == VINF_IOM_R3_IOPORT_READ)
8066	Assert(!fIOWrite);
8067	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8068	Assert(fIOWrite);
8069	else
8070	{
8071	AssertMsg( RT_FAILURE(rc)
8072	\|\| rc == VINF_SUCCESS
8073	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8074	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8075	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8076	}
8077	#endif
8078
8079	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8080	return rc;
8081	}
8082
8083
8084	/**
8085	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8086	* VM-exit.
8087	*/
8088	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8089	{
8090	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8091
8092	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8093	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8094	AssertRCReturn(rc, rc);
8095	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8096	{
8097	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8098	AssertRCReturn(rc, rc);
8099	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8100	{
8101	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8102	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8103	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8104	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8105	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8106	{
8107	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8108	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8109
8110	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8111	Assert(!pVCpu->hm.s.Event.fPending);
8112	pVCpu->hm.s.Event.fPending = true;
8113	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8114	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8115	AssertRCReturn(rc, rc);
8116	if (fErrorCodeValid)
8117	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8118	else
8119	pVCpu->hm.s.Event.u32ErrCode = 0;
8120	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8121	&& uVector == X86_XCPT_PF)
8122	{
8123	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8124	}
8125	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8126	}
8127	}
8128	}
8129	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8130	* emulation. */
8131	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8132	return VERR_EM_INTERPRETER;
8133	}
8134
8135
8136	/**
8137	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8138	*/
8139	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8140	{
8141	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8142	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8143	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8144	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8145	AssertRCReturn(rc, rc);
8146	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8147	return VINF_EM_DBG_STOP;
8148	}
8149
8150
8151	/**
8152	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8153	*/
8154	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8155	{
8156	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8157
8158	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8159	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8160	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8161	return VINF_SUCCESS;
8162	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8163	return rc;
8164
8165	#if 0
8166	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8167	* just sync the whole thing. */
8168	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8169	#else
8170	/* Aggressive state sync. for now. */
8171	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8172	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8173	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8174	#endif
8175	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8176	AssertRCReturn(rc, rc);
8177
8178	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8179	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8180	switch (uAccessType)
8181	{
8182	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8183	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8184	{
8185	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8186	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8187	{
8188	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8189	}
8190
8191	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8192	GCPhys &= PAGE_BASE_GC_MASK;
8193	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8194	PVM pVM = pVCpu->CTX_SUFF(pVM);
8195	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8196	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8197
8198	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8199	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8200	CPUMCTX2CORE(pMixedCtx), GCPhys);
8201	rc = VBOXSTRICTRC_VAL(rc2);
8202	Log(("ApicAccess rc=%d\n", rc));
8203	if ( rc == VINF_SUCCESS
8204	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8205	\|\| rc == VERR_PAGE_NOT_PRESENT)
8206	{
8207	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8208	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8209	rc = VINF_SUCCESS;
8210	}
8211	break;
8212	}
8213
8214	default:
8215	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8216	rc = VINF_EM_RAW_EMULATE_INSTR;
8217	break;
8218	}
8219
8220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8221	return rc;
8222	}
8223
8224
8225	/**
8226	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8227	* VM-exit.
8228	*/
8229	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8230	{
8231	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8232
8233	/* We should -not- get this VM-exit if the guest is debugging. */
8234	if (CPUMIsGuestDebugStateActive(pVCpu))
8235	{
8236	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8237	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8238	}
8239
8240	int rc = VERR_INTERNAL_ERROR_5;
8241	if ( !DBGFIsStepping(pVCpu)
8242	&& !CPUMIsHyperDebugStateActive(pVCpu))
8243	{
8244	/* Don't intercept MOV DRx. */
8245	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8246	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8247	AssertRCReturn(rc, rc);
8248
8249	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8250	PVM pVM = pVCpu->CTX_SUFF(pVM);
8251	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8252	AssertRC(rc);
8253	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8254
8255	#ifdef VBOX_WITH_STATISTICS
8256	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8257	AssertRCReturn(rc, rc);
8258	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8259	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8260	else
8261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8262	#endif
8263	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8264	return VINF_SUCCESS;
8265	}
8266
8267	/*
8268	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8269	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8270	*/
8271	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8272	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8273	AssertRCReturn(rc, rc);
8274
8275	PVM pVM = pVCpu->CTX_SUFF(pVM);
8276	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8277	{
8278	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8279	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8280	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8281	if (RT_SUCCESS(rc))
8282	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8283	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8284	}
8285	else
8286	{
8287	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8288	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8289	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8290	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8291	}
8292
8293	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8294	if (RT_SUCCESS(rc))
8295	{
8296	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8297	AssertRCReturn(rc2, rc2);
8298	}
8299	return rc;
8300	}
8301
8302
8303	/**
8304	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8305	* Conditional VM-exit.
8306	*/
8307	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8308	{
8309	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8310	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8311
8312	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8313	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8314	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8315	return VINF_SUCCESS;
8316	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8317	return rc;
8318
8319	RTGCPHYS GCPhys = 0;
8320	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8321
8322	#if 0
8323	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8324	#else
8325	/* Aggressive state sync. for now. */
8326	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8327	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8328	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8329	#endif
8330	AssertRCReturn(rc, rc);
8331
8332	/*
8333	* If we succeed, resume guest execution.
8334	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8335	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8336	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8337	* weird case. See @bugref{6043}.
8338	*/
8339	PVM pVM = pVCpu->CTX_SUFF(pVM);
8340	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8341	rc = VBOXSTRICTRC_VAL(rc2);
8342	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8343	if ( rc == VINF_SUCCESS
8344	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8345	\|\| rc == VERR_PAGE_NOT_PRESENT)
8346	{
8347	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8348	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8349	return VINF_SUCCESS;
8350	}
8351	return rc;
8352	}
8353
8354
8355	/**
8356	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8357	* VM-exit.
8358	*/
8359	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8360	{
8361	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8362	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8363
8364	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8365	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8366	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8367	return VINF_SUCCESS;
8368	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8369	return rc;
8370
8371	RTGCPHYS GCPhys = 0;
8372	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8373	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8374	#if 0
8375	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8376	#else
8377	/* Aggressive state sync. for now. */
8378	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8379	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8380	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8381	#endif
8382	AssertRCReturn(rc, rc);
8383
8384	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8385	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8386
8387	RTGCUINT uErrorCode = 0;
8388	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8389	uErrorCode \|= X86_TRAP_PF_ID;
8390	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8391	uErrorCode \|= X86_TRAP_PF_RW;
8392	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8393	uErrorCode \|= X86_TRAP_PF_P;
8394
8395	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8396
8397	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8398	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8399
8400	/* Handle the pagefault trap for the nested shadow table. */
8401	PVM pVM = pVCpu->CTX_SUFF(pVM);
8402	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8403	TRPMResetTrap(pVCpu);
8404
8405	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8406	if ( rc == VINF_SUCCESS
8407	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8408	\|\| rc == VERR_PAGE_NOT_PRESENT)
8409	{
8410	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8411	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8412	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8413	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8414	return VINF_SUCCESS;
8415	}
8416
8417	Log(("EPT return to ring-3 rc=%d\n"));
8418	return rc;
8419	}
8420
8421
8422	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8423	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8424	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8425	/**
8426	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8427	*/
8428	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8429	{
8430	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8431	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8432
8433	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8434	AssertRCReturn(rc, rc);
8435
8436	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8437	{
8438	/* Old-style FPU error reporting needs some extra work. */
8439	/** @todo don't fall back to the recompiler, but do it manually. */
8440	return VERR_EM_INTERPRETER;
8441	}
8442	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8443	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8444	return rc;
8445	}
8446
8447
8448	/**
8449	* VM-exit exception handler for #BP (Breakpoint exception).
8450	*/
8451	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8452	{
8453	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8455
8456	/** @todo Try optimize this by not saving the entire guest state unless
8457	* really needed. */
8458	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8459	AssertRCReturn(rc, rc);
8460
8461	PVM pVM = pVCpu->CTX_SUFF(pVM);
8462	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8463	if (rc == VINF_EM_RAW_GUEST_TRAP)
8464	{
8465	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8466	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8467	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8468	AssertRCReturn(rc, rc);
8469
8470	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8471	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8472	}
8473
8474	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8475	return rc;
8476	}
8477
8478
8479	/**
8480	* VM-exit exception handler for #DB (Debug exception).
8481	*/
8482	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8483	{
8484	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8486
8487	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8488	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8489	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8490	AssertRCReturn(rc, rc);
8491
8492	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8493	uint64_t uDR6 = X86_DR6_INIT_VAL;
8494	uDR6 \|= (pVmxTransient->uExitQualification
8495	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8496	PVM pVM = pVCpu->CTX_SUFF(pVM);
8497	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8498	if (rc == VINF_EM_RAW_GUEST_TRAP)
8499	{
8500	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8501	pMixedCtx->dr[6] = uDR6;
8502
8503	if (CPUMIsGuestDebugStateActive(pVCpu))
8504	ASMSetDR6(pMixedCtx->dr[6]);
8505
8506	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8507
8508	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8509	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8510
8511	/* Paranoia. */
8512	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8513	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8514	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8515
8516	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8517	AssertRCReturn(rc,rc);
8518
8519	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8520	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8521	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8522	AssertRCReturn(rc2, rc2);
8523	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8524	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8525	rc = VINF_SUCCESS;
8526	}
8527
8528	return rc;
8529	}
8530
8531
8532	/**
8533	* VM-exit exception handler for #NM (Device-not-available exception: floating
8534	* point exception).
8535	*/
8536	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8537	{
8538	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8539
8540	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8541	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8542	#endif
8543
8544	/* We require CR0 and EFER. EFER is always up-to-date. */
8545	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8546	AssertRCReturn(rc, rc);
8547
8548	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8549	PVM pVM = pVCpu->CTX_SUFF(pVM);
8550	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8551	if (rc == VINF_SUCCESS)
8552	{
8553	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8554	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8556	return VINF_SUCCESS;
8557	}
8558
8559	/* Forward #NM to the guest. */
8560	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8561	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8562	AssertRCReturn(rc, rc);
8563	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8564	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8566	return rc;
8567	}
8568
8569
8570	/**
8571	* VM-exit exception handler for #GP (General-protection exception).
8572	*
8573	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8574	*/
8575	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8576	{
8577	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8578	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8579
8580	int rc = VERR_INTERNAL_ERROR_5;
8581	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8582	{
8583	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8584	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8585	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8586	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8587	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8588	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8589	AssertRCReturn(rc, rc);
8590	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8591	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8592	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8593	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8594	return rc;
8595	#else
8596	/* We don't intercept #GP. */
8597	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8598	return VERR_VMX_UNEXPECTED_EXCEPTION;
8599	#endif
8600	}
8601
8602	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8603	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8604
8605	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8606	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8607	AssertRCReturn(rc, rc);
8608
8609	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8610	uint32_t cbOp = 0;
8611	PVM pVM = pVCpu->CTX_SUFF(pVM);
8612	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8613	if (RT_SUCCESS(rc))
8614	{
8615	rc = VINF_SUCCESS;
8616	Assert(cbOp == pDis->cbInstr);
8617	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8618	switch (pDis->pCurInstr->uOpcode)
8619	{
8620	case OP_CLI:
8621	pMixedCtx->eflags.Bits.u1IF = 0;
8622	pMixedCtx->rip += pDis->cbInstr;
8623	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8625	break;
8626
8627	case OP_STI:
8628	pMixedCtx->eflags.Bits.u1IF = 1;
8629	pMixedCtx->rip += pDis->cbInstr;
8630	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8631	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8632	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8633	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8634	break;
8635
8636	case OP_HLT:
8637	rc = VINF_EM_HALT;
8638	pMixedCtx->rip += pDis->cbInstr;
8639	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8641	break;
8642
8643	case OP_POPF:
8644	{
8645	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8646	uint32_t cbParm = 0;
8647	uint32_t uMask = 0;
8648	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8649	{
8650	cbParm = 4;
8651	uMask = 0xffffffff;
8652	}
8653	else
8654	{
8655	cbParm = 2;
8656	uMask = 0xffff;
8657	}
8658
8659	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8660	RTGCPTR GCPtrStack = 0;
8661	X86EFLAGS uEflags;
8662	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8663	&GCPtrStack);
8664	if (RT_SUCCESS(rc))
8665	{
8666	Assert(sizeof(uEflags.u32) >= cbParm);
8667	uEflags.u32 = 0;
8668	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8669	}
8670	if (RT_FAILURE(rc))
8671	{
8672	rc = VERR_EM_INTERPRETER;
8673	break;
8674	}
8675	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8676	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8677	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8678	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8679	pMixedCtx->eflags.Bits.u1RF = 0;
8680	pMixedCtx->esp += cbParm;
8681	pMixedCtx->esp &= uMask;
8682	pMixedCtx->rip += pDis->cbInstr;
8683	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8684	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8685	break;
8686	}
8687
8688	case OP_PUSHF:
8689	{
8690	uint32_t cbParm = 0;
8691	uint32_t uMask = 0;
8692	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8693	{
8694	cbParm = 4;
8695	uMask = 0xffffffff;
8696	}
8697	else
8698	{
8699	cbParm = 2;
8700	uMask = 0xffff;
8701	}
8702
8703	/* Get the stack pointer & push the contents of eflags onto the stack. */
8704	RTGCPTR GCPtrStack = 0;
8705	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8706	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8707	if (RT_FAILURE(rc))
8708	{
8709	rc = VERR_EM_INTERPRETER;
8710	break;
8711	}
8712	X86EFLAGS uEflags;
8713	uEflags = pMixedCtx->eflags;
8714	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8715	uEflags.Bits.u1RF = 0;
8716	uEflags.Bits.u1VM = 0;
8717
8718	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8719	if (RT_FAILURE(rc))
8720	{
8721	rc = VERR_EM_INTERPRETER;
8722	break;
8723	}
8724	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8725	pMixedCtx->esp -= cbParm;
8726	pMixedCtx->esp &= uMask;
8727	pMixedCtx->rip += pDis->cbInstr;
8728	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8730	break;
8731	}
8732
8733	case OP_IRET:
8734	{
8735	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8736	* instruction reference. */
8737	RTGCPTR GCPtrStack = 0;
8738	uint32_t uMask = 0xffff;
8739	uint16_t aIretFrame[3];
8740	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8741	{
8742	rc = VERR_EM_INTERPRETER;
8743	break;
8744	}
8745	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8746	&GCPtrStack);
8747	if (RT_SUCCESS(rc))
8748	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8749	if (RT_FAILURE(rc))
8750	{
8751	rc = VERR_EM_INTERPRETER;
8752	break;
8753	}
8754	pMixedCtx->eip = 0;
8755	pMixedCtx->ip = aIretFrame[0];
8756	pMixedCtx->cs.Sel = aIretFrame[1];
8757	pMixedCtx->cs.ValidSel = aIretFrame[1];
8758	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8759	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8760	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8761	pMixedCtx->sp += sizeof(aIretFrame);
8762	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8763	\| HM_CHANGED_GUEST_RFLAGS;
8764	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8765	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8766	break;
8767	}
8768
8769	case OP_INT:
8770	{
8771	uint16_t uVector = pDis->Param1.uValue & 0xff;
8772	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8773	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8774	break;
8775	}
8776
8777	case OP_INTO:
8778	{
8779	if (pMixedCtx->eflags.Bits.u1OF)
8780	{
8781	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8783	}
8784	break;
8785	}
8786
8787	default:
8788	{
8789	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8790	EMCODETYPE_SUPERVISOR);
8791	rc = VBOXSTRICTRC_VAL(rc2);
8792	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8793	Log(("#GP rc=%Rrc\n", rc));
8794	break;
8795	}
8796	}
8797	}
8798	else
8799	rc = VERR_EM_INTERPRETER;
8800
8801	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8802	("#GP Unexpected rc=%Rrc\n", rc));
8803	return rc;
8804	}
8805
8806
8807	/**
8808	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8809	* the exception reported in the VMX transient structure back into the VM.
8810	*
8811	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8812	* VMX transient structure to be up-to-date.
8813	*/
8814	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8815	{
8816	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8817
8818	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8819	hmR0VmxCheckExitDueToEventDelivery(). */
8820	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8821	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8822	AssertRCReturn(rc, rc);
8823	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8824	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8825	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8826	return VINF_SUCCESS;
8827	}
8828
8829
8830	/**
8831	* VM-exit exception handler for #PF (Page-fault exception).
8832	*/
8833	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8834	{
8835	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8836	PVM pVM = pVCpu->CTX_SUFF(pVM);
8837	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8838	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8839	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8840	AssertRCReturn(rc, rc);
8841
8842	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8843	if (pVM->hm.s.fNestedPaging)
8844	{
8845	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8846	{
8847	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8848	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8849	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8850	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8851	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8852	}
8853	else
8854	{
8855	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8856	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8857	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8858	Log(("Pending #DF due to vectoring #PF. NP\n"));
8859	}
8860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8861	return rc;
8862	}
8863	#else
8864	Assert(!pVM->hm.s.fNestedPaging);
8865	#endif
8866
8867	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8868	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8869	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8870	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8871	AssertRCReturn(rc, rc);
8872	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8873	if ( pVM->hm.s.fTRPPatchingAllowed
8874	&& pVM->hm.s.pGuestPatchMem
8875	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8876	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8877	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8878	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8879	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8880	{
8881	RTGCPHYS GCPhys;
8882	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8883	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8884	if ( rc == VINF_SUCCESS
8885	&& GCPhys == GCPhysApicBase)
8886	{
8887	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8888	AssertRCReturn(rc, rc);
8889
8890	/* Only attempt to patch the instruction once. */
8891	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8892	if (!pPatch)
8893	return VINF_EM_HM_PATCH_TPR_INSTR;
8894	}
8895	}
8896	#endif
8897
8898	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8899	AssertRCReturn(rc, rc);
8900
8901	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8902	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8903
8904	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8905	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8906	(RTGCPTR)pVmxTransient->uExitQualification);
8907
8908	Log(("#PF: rc=%Rrc\n", rc));
8909	if (rc == VINF_SUCCESS)
8910	{
8911	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8912	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8913	* memory? We don't update the whole state here... */
8914	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8915	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8916	TRPMResetTrap(pVCpu);
8917	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8918	return rc;
8919	}
8920	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8921	{
8922	if (!pVmxTransient->fVectoringPF)
8923	{
8924	/* It's a guest page fault and needs to be reflected to the guest. */
8925	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8926	TRPMResetTrap(pVCpu);
8927	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8928	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8929	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8930	0 /* cbInstr */, uGstErrorCode, pMixedCtx->cr2);
8931	}
8932	else
8933	{
8934	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8935	TRPMResetTrap(pVCpu);
8936	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8937	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8938	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8939	}
8940
8941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8942	return VINF_SUCCESS;
8943	}
8944
8945	TRPMResetTrap(pVCpu);
8946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8947	return rc;
8948	}
8949

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45676

Download in other formats: