HMVMXR0.cpp@ 45685

Last change on this file since 45685 was 45685, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Safer to update CR2 while injecting exceptions and not earlier while converting from TRPM traps (from ring-3).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 373.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45685 2013-04-23 17:27:33Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_TRAP_ALL_EXCEPTIONS
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define HMVMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define HMVMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/** Use the function table. */
56	#define HMVMX_USE_FUNCTION_TABLE
57
58	/** This bit indicates the segment selector is unusable in VT-x. */
59	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
60
61	/** Determine which tagged-TLB flush handler to use. */
62	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67	/**
68	* Updated-guest-state flags.
69	*/
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DEBUG \
102	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
103	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110
111	/**
112	* Flags to skip redundant reads of some common VMCS fields that are not part of
113	* the guest-CPU state but are in the transient structure.
114	*/
115	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121
122	/**
123	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
124	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
125	* we have Nested Paging support.
126	*/
127	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
128	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
129	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
130	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
131	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
133	\| RT_BIT(X86_XCPT_XF))
134
135	/**
136	* Exception bitmap mask for all contributory exceptions.
137	*/
138	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
139	\| RT_BIT(X86_XCPT_DE))
140
141	/** Maximum VM-instruction error number. */
142	#define HMVMX_INSTR_ERROR_MAX 28
143
144
145	/*******************************************************************************
146	* Structures and Typedefs *
147	*******************************************************************************/
148	/**
149	* A state structure for holding miscellaneous information across
150	* VMX non-root operation and restored after the transition.
151	*/
152	typedef struct VMXTRANSIENT
153	{
154	/** The host's rflags/eflags. */
155	RTCCUINTREG uEFlags;
156	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
157	uint64_t u64LStarMsr;
158	/** The guest's TPR value used for TPR shadowing. */
159	uint8_t u8GuestTpr;
160
161	/** The basic VM-exit reason. */
162	uint16_t uExitReason;
163	/** The VM-exit exit qualification. */
164	RTGCUINTPTR uExitQualification;
165	/** The VM-exit interruption error code. */
166	uint32_t uExitIntrErrorCode;
167
168	/** The VM-exit interruption-information field. */
169	uint32_t uExitIntrInfo;
170	/** Whether the VM-entry failed or not. */
171	bool fVMEntryFailed;
172	/** The VM-exit instruction-length field. */
173	uint32_t cbInstr;
174
175	/** The VM-entry interruption-information field. */
176	uint32_t uEntryIntrInfo;
177	/** The VM-entry exception error code field. */
178	uint32_t uEntryXcptErrorCode;
179	/** The VM-entry instruction length field. */
180	uint32_t cbEntryInstr;
181
182	/** IDT-vectoring information field. */
183	uint32_t uIdtVectoringInfo;
184	/** IDT-vectoring error code. */
185	uint32_t uIdtVectoringErrorCode;
186
187	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
188	uint32_t fVmcsFieldsRead;
189	/** Whether TSC-offsetting should be setup before VM-entry. */
190	bool fUpdateTscOffsettingAndPreemptTimer;
191	/** Whether the VM-exit was caused by a page-fault during delivery of a
192	* contributary exception or a page-fault. */
193	bool fVectoringPF;
194	} VMXTRANSIENT, *PVMXTRANSIENT;
195
196	/**
197	* MSR-bitmap read permissions.
198	*/
199	typedef enum VMXMSREXITREAD
200	{
201	/** Reading this MSR causes a VM-exit. */
202	VMXMSREXIT_INTERCEPT_READ = 0xb,
203	/** Reading this MSR does not cause a VM-exit. */
204	VMXMSREXIT_PASSTHRU_READ
205	} VMXMSREXITREAD;
206
207	/**
208	* MSR-bitmap write permissions.
209	*/
210	typedef enum VMXMSREXITWRITE
211	{
212	/** Writing to this MSR causes a VM-exit. */
213	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
214	/** Writing to this MSR does not cause a VM-exit. */
215	VMXMSREXIT_PASSTHRU_WRITE
216	} VMXMSREXITWRITE;
217
218
219	/*******************************************************************************
220	* Internal Functions *
221	*******************************************************************************/
222	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
223	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
224	uint32_t u32ErrCode, RTGCUINTREG uCR2, uint32_t *puIntrState);
225	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
226	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
227	#endif
228	#ifndef HMVMX_USE_FUNCTION_TABLE
229	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
230	#endif
231
232	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276
277	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284
285
286	/*******************************************************************************
287	* Global Variables *
288	*******************************************************************************/
289	#ifdef HMVMX_USE_FUNCTION_TABLE
290	/**
291	* VM-exit handler.
292	*
293	* @returns VBox status code.
294	* @param pVCpu Pointer to the VMCPU.
295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
296	* out-of-sync. Make sure to update the required
297	* fields before using them.
298	* @param pVmxTransient Pointer to the VMX-transient structure.
299	*/
300	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	/** Pointer to VM-exit handler. */
302	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
303
304	/**
305	* VMX_EXIT dispatch table.
306	*/
307	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
308	{
309	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
310	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
311	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
312	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
313	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
314	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
315	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
316	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
317	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
318	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
319	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
320	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
321	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
322	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
323	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
324	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
325	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
326	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
327	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
328	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
329	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
330	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
331	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
332	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
333	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
334	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
335	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
336	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
337	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
338	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
339	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
340	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
341	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
342	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
343	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
344	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
345	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
346	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
347	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
348	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
349	/* 40 UNDEFINED */ hmR0VmxExitPause,
350	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
351	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
352	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
353	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
354	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
355	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
356	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
357	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
358	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
359	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
360	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
361	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
362	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
363	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
364	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
365	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
366	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
367	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
368	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
369	};
370	#endif /* HMVMX_USE_FUNCTION_TABLE */
371
372	#ifdef VBOX_STRICT
373	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
374	{
375	/* 0 */ "(Not Used)",
376	/* 1 */ "VMCALL executed in VMX root operation.",
377	/* 2 */ "VMCLEAR with invalid physical address.",
378	/* 3 */ "VMCLEAR with VMXON pointer.",
379	/* 4 */ "VMLAUNCH with non-clear VMCS.",
380	/* 5 */ "VMRESUME with non-launched VMCS.",
381	/* 6 */ "VMRESUME after VMXOFF",
382	/* 7 */ "VM entry with invalid control fields.",
383	/* 8 */ "VM entry with invalid host state fields.",
384	/* 9 */ "VMPTRLD with invalid physical address.",
385	/* 10 */ "VMPTRLD with VMXON pointer.",
386	/* 11 */ "VMPTRLD with incorrect revision identifier.",
387	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
388	/* 13 */ "VMWRITE to read-only VMCS component.",
389	/* 14 */ "(Not Used)",
390	/* 15 */ "VMXON executed in VMX root operation.",
391	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
392	/* 17 */ "VM entry with non-launched executing VMCS.",
393	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
394	/* 19 */ "VMCALL with non-clear VMCS.",
395	/* 20 */ "VMCALL with invalid VM-exit control fields.",
396	/* 21 */ "(Not Used)",
397	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
398	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
399	/* 24 */ "VMCALL with invalid SMM-monitor features.",
400	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
401	/* 26 */ "VM entry with events blocked by MOV SS.",
402	/* 27 */ "(Not Used)",
403	/* 28 */ "Invalid operand to INVEPT/INVVPID."
404	};
405	#endif /* VBOX_STRICT */
406
407
408
409	/**
410	* Updates the VM's last error record. If there was a VMX instruction error,
411	* reads the error data from the VMCS and updates VCPU's last error record as
412	* well.
413	*
414	* @param pVM Pointer to the VM.
415	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
416	* VERR_VMX_UNABLE_TO_START_VM or
417	* VERR_VMX_INVALID_VMCS_FIELD).
418	* @param rc The error code.
419	*/
420	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
421	{
422	AssertPtr(pVM);
423	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
424	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
425	{
426	AssertPtrReturnVoid(pVCpu);
427	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
428	}
429	pVM->hm.s.lLastError = rc;
430	}
431
432
433	/**
434	* Reads the VM-entry interruption-information field from the VMCS into the VMX
435	* transient structure.
436	*
437	* @returns VBox status code.
438	* @param pVmxTransient Pointer to the VMX transient structure.
439	*
440	* @remarks No-long-jump zone!!!
441	*/
442	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
443	{
444	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
445	AssertRCReturn(rc, rc);
446	return VINF_SUCCESS;
447	}
448
449
450	/**
451	* Reads the VM-entry exception error code field from the VMCS into
452	* the VMX transient structure.
453	*
454	* @returns VBox status code.
455	* @param pVmxTransient Pointer to the VMX transient structure.
456	*
457	* @remarks No-long-jump zone!!!
458	*/
459	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
460	{
461	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
462	AssertRCReturn(rc, rc);
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Reads the VM-entry exception error code field from the VMCS into
469	* the VMX transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVCpu Pointer to the VMCPU.
473	* @param pVmxTransient Pointer to the VMX transient structure.
474	*
475	* @remarks No-long-jump zone!!!
476	*/
477	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
478	{
479	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
480	AssertRCReturn(rc, rc);
481	return VINF_SUCCESS;
482	}
483
484
485	/**
486	* Reads the VM-exit interruption-information field from the VMCS into the VMX
487	* transient structure.
488	*
489	* @returns VBox status code.
490	* @param pVCpu Pointer to the VMCPU.
491	* @param pVmxTransient Pointer to the VMX transient structure.
492	*/
493	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
494	{
495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
496	{
497	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
498	AssertRCReturn(rc, rc);
499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
500	}
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption error code from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit instruction length field from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the exit qualification from the VMCS into the VMX transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVCpu Pointer to the VMCPU.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*/
552	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
553	{
554	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
555	{
556	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
557	AssertRCReturn(rc, rc);
558	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
559	}
560	return VINF_SUCCESS;
561	}
562
563
564	/**
565	* Reads the IDT-vectoring information field from the VMCS into the VMX
566	* transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVmxTransient Pointer to the VMX transient structure.
570	*
571	* @remarks No-long-jump zone!!!
572	*/
573	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
574	{
575	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
576	{
577	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
578	AssertRCReturn(rc, rc);
579	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
580	}
581	return VINF_SUCCESS;
582	}
583
584
585	/**
586	* Reads the IDT-vectoring error code from the VMCS into the VMX
587	* transient structure.
588	*
589	* @returns VBox status code.
590	* @param pVmxTransient Pointer to the VMX transient structure.
591	*/
592	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
593	{
594	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
595	{
596	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
597	AssertRCReturn(rc, rc);
598	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
599	}
600	return VINF_SUCCESS;
601	}
602
603
604	/**
605	* Enters VMX root mode operation on the current CPU.
606	*
607	* @returns VBox status code.
608	* @param pVM Pointer to the VM (optional, can be NULL, after
609	* a resume).
610	* @param HCPhysCpuPage Physical address of the VMXON region.
611	* @param pvCpuPage Pointer to the VMXON region.
612	*/
613	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
614	{
615	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
616	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
617
618	if (pVM)
619	{
620	/* Write the VMCS revision dword to the VMXON region. */
621	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
622	}
623
624	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
625	RTCCUINTREG fFlags = ASMIntDisableFlags();
626
627	/* Enable the VMX bit in CR4 if necessary. */
628	RTCCUINTREG uCr4 = ASMGetCR4();
629	if (!(uCr4 & X86_CR4_VMXE))
630	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
631
632	/* Enter VMX root mode. */
633	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
634	if (RT_FAILURE(rc))
635	ASMSetCR4(uCr4);
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Exits VMX root mode operation on the current CPU.
645	*
646	* @returns VBox status code.
647	*/
648	static int hmR0VmxLeaveRootMode(void)
649	{
650	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
651	RTCCUINTREG fFlags = ASMIntDisableFlags();
652	int rc = VINF_SUCCESS;
653
654	/* If we're for some reason not in VMX root mode, then don't leave it. */
655	if (ASMGetCR4() & X86_CR4_VMXE)
656	{
657	/* Exit VMX root mode and clear the VMX bit in CR4 */
658	VMXDisable();
659	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
660	}
661	else
662	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
663
664	/* Restore interrupts. */
665	ASMSetFlags(fFlags);
666	return rc;
667	}
668
669
670	/**
671	* Allocates and maps one physically contiguous page. The allocated page is
672	* zero'd out. (Used by various VT-x structures).
673	*
674	* @returns IPRT status code.
675	* @param pMemObj Pointer to the ring-0 memory object.
676	* @param ppVirt Where to store the virtual address of the
677	* allocation.
678	* @param pPhys Where to store the physical address of the
679	* allocation.
680	*/
681	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
682	{
683	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
684	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
685	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
686
687	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
688	if (RT_FAILURE(rc))
689	return rc;
690	ppVirt = RTR0MemObjAddress(pMemObj);
691	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
692	ASMMemZero32(*ppVirt, PAGE_SIZE);
693	return VINF_SUCCESS;
694	}
695
696
697	/**
698	* Frees and unmaps an allocated physical page.
699	*
700	* @param pMemObj Pointer to the ring-0 memory object.
701	* @param ppVirt Where to re-initialize the virtual address of
702	* allocation as 0.
703	* @param pHCPhys Where to re-initialize the physical address of the
704	* allocation as 0.
705	*/
706	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
707	{
708	AssertPtr(pMemObj);
709	AssertPtr(ppVirt);
710	AssertPtr(pHCPhys);
711	if (*pMemObj != NIL_RTR0MEMOBJ)
712	{
713	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
714	AssertRC(rc);
715	*pMemObj = NIL_RTR0MEMOBJ;
716	*ppVirt = 0;
717	*pHCPhys = 0;
718	}
719	}
720
721
722	/**
723	* Worker function to free VT-x related structures.
724	*
725	* @returns IPRT status code.
726	* @param pVM Pointer to the VM.
727	*/
728	static void hmR0VmxStructsFree(PVM pVM)
729	{
730	for (VMCPUID i = 0; i < pVM->cCpus; i++)
731	{
732	PVMCPU pVCpu = &pVM->aCpus[i];
733	AssertPtr(pVCpu);
734
735	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
736	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
737
738	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
739	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
740
741	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
742	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
743	}
744
745	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
748	#endif
749	}
750
751
752	/**
753	* Worker function to allocate VT-x related VM structures.
754	*
755	* @returns IPRT status code.
756	* @param pVM Pointer to the VM.
757	*/
758	static int hmR0VmxStructsAlloc(PVM pVM)
759	{
760	/*
761	* Initialize members up-front so we can cleanup properly on allocation failure.
762	*/
763	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
764	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
765	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
766	pVM->hm.s.vmx.HCPhys##a_Name = 0;
767
768	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
769	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
770	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
771	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
772
773	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
774	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
775	#endif
776	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
777
778	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
779	for (VMCPUID i = 0; i < pVM->cCpus; i++)
780	{
781	PVMCPU pVCpu = &pVM->aCpus[i];
782	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
783	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
784	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
785	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
786	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
787	}
788	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
789	#undef VMXLOCAL_INIT_VM_MEMOBJ
790
791	/*
792	* Allocate all the VT-x structures.
793	*/
794	int rc = VINF_SUCCESS;
795	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
796	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
797	if (RT_FAILURE(rc))
798	goto cleanup;
799	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
800	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
801	#endif
802
803	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
804	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
805	{
806	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
807	&pVM->hm.s.vmx.HCPhysApicAccess);
808	if (RT_FAILURE(rc))
809	goto cleanup;
810	}
811
812	/*
813	* Initialize per-VCPU VT-x structures.
814	*/
815	for (VMCPUID i =0; i < pVM->cCpus; i++)
816	{
817	PVMCPU pVCpu = &pVM->aCpus[i];
818	AssertPtr(pVCpu);
819
820	/* Allocate the VM control structure (VMCS). */
821	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
822	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825
826	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
827	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
828	{
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
830	&pVCpu->hm.s.vmx.HCPhysVirtApic);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	}
834
835	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
836	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
837	{
838	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
839	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
840	if (RT_FAILURE(rc))
841	goto cleanup;
842	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
843	}
844
845	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
846	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
847	if (RT_FAILURE(rc))
848	goto cleanup;
849
850	/* Allocate the VM-exit MSR-load page for the host MSRs. */
851	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
852	if (RT_FAILURE(rc))
853	goto cleanup;
854	}
855
856	return VINF_SUCCESS;
857
858	cleanup:
859	hmR0VmxStructsFree(pVM);
860	return rc;
861	}
862
863
864	/**
865	* Does global VT-x initialization (called during module initialization).
866	*
867	* @returns VBox status code.
868	*/
869	VMMR0DECL(int) VMXR0GlobalInit(void)
870	{
871	#ifdef HMVMX_USE_FUNCTION_TABLE
872	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
873	# ifdef VBOX_STRICT
874	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
875	Assert(g_apfnVMExitHandlers[i]);
876	# endif
877	#endif
878	return VINF_SUCCESS;
879	}
880
881
882	/**
883	* Does global VT-x termination (called during module termination).
884	*/
885	VMMR0DECL(void) VMXR0GlobalTerm()
886	{
887	/* Nothing to do currently. */
888	}
889
890
891	/**
892	* Sets up and activates VT-x on the current CPU.
893	*
894	* @returns VBox status code.
895	* @param pCpu Pointer to the global CPU info struct.
896	* @param pVM Pointer to the VM (can be NULL after a host resume
897	* operation).
898	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
899	* fEnabledByHost is true).
900	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
901	* @a fEnabledByHost is true).
902	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
903	* enable VT-x/AMD-V on the host.
904	*/
905	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
906	{
907	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
908	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
909
910	if (!fEnabledByHost)
911	{
912	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
913	if (RT_FAILURE(rc))
914	return rc;
915	}
916
917	/*
918	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
919	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
920	* each time while reusing a VPID after hitting the MaxASID limit once.
921	*/
922	if ( pVM
923	&& pVM->hm.s.vmx.fVpid
924	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
925	{
926	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
927	pCpu->fFlushAsidBeforeUse = false;
928	}
929	else
930	pCpu->fFlushAsidBeforeUse = true;
931
932	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
933	++pCpu->cTlbFlushes;
934
935	return VINF_SUCCESS;
936	}
937
938
939	/**
940	* Deactivates VT-x on the current CPU.
941	*
942	* @returns VBox status code.
943	* @param pCpu Pointer to the global CPU info struct.
944	* @param pvCpuPage Pointer to the VMXON region.
945	* @param HCPhysCpuPage Physical address of the VMXON region.
946	*/
947	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
948	{
949	NOREF(pCpu);
950	NOREF(pvCpuPage);
951	NOREF(HCPhysCpuPage);
952
953	hmR0VmxLeaveRootMode();
954	return VINF_SUCCESS;
955	}
956
957
958	/**
959	* Sets the permission bits for the specified MSR in the MSR bitmap.
960	*
961	* @param pVCpu Pointer to the VMCPU.
962	* @param uMSR The MSR value.
963	* @param enmRead Whether reading this MSR causes a VM-exit.
964	* @param enmWrite Whether writing this MSR causes a VM-exit.
965	*/
966	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
967	{
968	int32_t iBit;
969	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
970
971	/*
972	* Layout:
973	* 0x000 - 0x3ff - Low MSR read bits
974	* 0x400 - 0x7ff - High MSR read bits
975	* 0x800 - 0xbff - Low MSR write bits
976	* 0xc00 - 0xfff - High MSR write bits
977	*/
978	if (uMsr <= 0x00001FFF)
979	iBit = uMsr;
980	else if ( uMsr >= 0xC0000000
981	&& uMsr <= 0xC0001FFF)
982	{
983	iBit = (uMsr - 0xC0000000);
984	pbMsrBitmap += 0x400;
985	}
986	else
987	{
988	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
989	return;
990	}
991
992	Assert(iBit <= 0x1fff);
993	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
994	ASMBitSet(pbMsrBitmap, iBit);
995	else
996	ASMBitClear(pbMsrBitmap, iBit);
997
998	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
999	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1000	else
1001	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1002	}
1003
1004
1005	/**
1006	* Flushes the TLB using EPT.
1007	*
1008	* @returns VBox status code.
1009	* @param pVM Pointer to the VM.
1010	* @param pVCpu Pointer to the VMCPU.
1011	* @param enmFlush Type of flush.
1012	*/
1013	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1014	{
1015	AssertPtr(pVM);
1016	Assert(pVM->hm.s.fNestedPaging);
1017
1018	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1019
1020	uint64_t descriptor[2];
1021	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1022	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1023
1024	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1025	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1026	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1027	}
1028
1029
1030	/**
1031	* Flushes the TLB using VPID.
1032	*
1033	* @returns VBox status code.
1034	* @param pVM Pointer to the VM.
1035	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1036	* enmFlush).
1037	* @param enmFlush Type of flush.
1038	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1039	* on @a enmFlush).
1040	*/
1041	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1042	{
1043	AssertPtr(pVM);
1044	Assert(pVM->hm.s.vmx.fVpid);
1045
1046	uint64_t descriptor[2];
1047	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1048	{
1049	descriptor[0] = 0;
1050	descriptor[1] = 0;
1051	}
1052	else
1053	{
1054	AssertPtr(pVCpu);
1055	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1056	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1057	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1058	descriptor[1] = GCPtr;
1059	}
1060
1061	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1062	AssertMsg(rc == VINF_SUCCESS,
1063	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1064	if ( RT_SUCCESS(rc)
1065	&& pVCpu)
1066	{
1067	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1068	}
1069	}
1070
1071
1072	/**
1073	* Invalidates a guest page by guest virtual address. Only relevant for
1074	* EPT/VPID, otherwise there is nothing really to invalidate.
1075	*
1076	* @returns VBox status code.
1077	* @param pVM Pointer to the VM.
1078	* @param pVCpu Pointer to the VMCPU.
1079	* @param GCVirt Guest virtual address of the page to invalidate.
1080	*/
1081	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1082	{
1083	AssertPtr(pVM);
1084	AssertPtr(pVCpu);
1085	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1086
1087	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1088	if (!fFlushPending)
1089	{
1090	/*
1091	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1092	* See @bugref{6043} and @bugref{6177}.
1093	*
1094	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1095	* function maybe called in a loop with individual addresses.
1096	*/
1097	if (pVM->hm.s.vmx.fVpid)
1098	{
1099	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1100	{
1101	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1102	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1103	}
1104	else
1105	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1106	}
1107	else if (pVM->hm.s.fNestedPaging)
1108	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1109	}
1110
1111	return VINF_SUCCESS;
1112	}
1113
1114
1115	/**
1116	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1117	* otherwise there is nothing really to invalidate.
1118	*
1119	* @returns VBox status code.
1120	* @param pVM Pointer to the VM.
1121	* @param pVCpu Pointer to the VMCPU.
1122	* @param GCPhys Guest physical address of the page to invalidate.
1123	*/
1124	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1125	{
1126	LogFlowFunc(("%RGp\n", GCPhys));
1127
1128	/*
1129	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1130	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1131	* This function might be called in a loop.
1132	*/
1133	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1134	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1135	return VINF_SUCCESS;
1136	}
1137
1138
1139	/**
1140	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1141	* case where neither EPT nor VPID is supported by the CPU.
1142	*
1143	* @param pVM Pointer to the VM.
1144	* @param pVCpu Pointer to the VMCPU.
1145	*
1146	* @remarks Called with interrupts disabled.
1147	*/
1148	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1149	{
1150	NOREF(pVM);
1151	AssertPtr(pVCpu);
1152	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1153	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1154
1155	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1156	AssertPtr(pCpu);
1157
1158	pVCpu->hm.s.TlbShootdown.cPages = 0;
1159	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1160	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1161	pVCpu->hm.s.fForceTLBFlush = false;
1162	return;
1163	}
1164
1165
1166	/**
1167	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	* @remarks All references to "ASID" in this function pertains to "VPID" in
1172	* Intel's nomenclature. The reason is, to avoid confusion in compare
1173	* statements since the host-CPU copies are named "ASID".
1174	*
1175	* @remarks Called with interrupts disabled.
1176	*/
1177	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1178	{
1179	AssertPtr(pVM);
1180	AssertPtr(pVCpu);
1181	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1182	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1183	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1184
1185	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1186	AssertPtr(pCpu);
1187
1188	/*
1189	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1190	* This can happen both for start & resume due to long jumps back to ring-3.
1191	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1192	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1193	*/
1194	bool fNewASID = false;
1195	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1196	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1197	{
1198	pVCpu->hm.s.fForceTLBFlush = true;
1199	fNewASID = true;
1200	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1201	}
1202
1203	/*
1204	* Check for explicit TLB shootdowns.
1205	*/
1206	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1207	{
1208	pVCpu->hm.s.fForceTLBFlush = true;
1209	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1210	}
1211
1212	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1213	if (pVCpu->hm.s.fForceTLBFlush)
1214	{
1215	if (fNewASID)
1216	{
1217	++pCpu->uCurrentAsid;
1218	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1219	{
1220	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1221	pCpu->cTlbFlushes++;
1222	pCpu->fFlushAsidBeforeUse = true;
1223	}
1224
1225	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1226	if (pCpu->fFlushAsidBeforeUse)
1227	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1228	}
1229	else
1230	{
1231	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1232	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1233	else
1234	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1235	}
1236
1237	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1238	pVCpu->hm.s.fForceTLBFlush = false;
1239	}
1240	else
1241	{
1242	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1243	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1244	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1245	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1246
1247	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1248	* not be executed. See hmQueueInvlPage() where it is commented
1249	* out. Support individual entry flushing someday. */
1250	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1251	{
1252	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1253
1254	/*
1255	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1256	* as supported by the CPU.
1257	*/
1258	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1259	{
1260	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1261	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1262	}
1263	else
1264	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1265	}
1266	else
1267	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1268	}
1269	pVCpu->hm.s.TlbShootdown.cPages = 0;
1270	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1271
1272	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1273	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1274	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1275	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1276	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1277	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1278
1279	/* Update VMCS with the VPID. */
1280	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1281	AssertRC(rc);
1282	}
1283
1284
1285	/**
1286	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1287	*
1288	* @returns VBox status code.
1289	* @param pVM Pointer to the VM.
1290	* @param pVCpu Pointer to the VMCPU.
1291	*
1292	* @remarks Called with interrupts disabled.
1293	*/
1294	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1295	{
1296	AssertPtr(pVM);
1297	AssertPtr(pVCpu);
1298	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1299	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1300
1301	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1302	AssertPtr(pCpu);
1303
1304	/*
1305	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1306	* This can happen both for start & resume due to long jumps back to ring-3.
1307	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1308	*/
1309	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1310	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1311	{
1312	pVCpu->hm.s.fForceTLBFlush = true;
1313	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1314	}
1315
1316	/* Check for explicit TLB shootdown flushes. */
1317	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1318	{
1319	pVCpu->hm.s.fForceTLBFlush = true;
1320	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1321	}
1322
1323	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1324	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1325
1326	if (pVCpu->hm.s.fForceTLBFlush)
1327	{
1328	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1329	pVCpu->hm.s.fForceTLBFlush = false;
1330	}
1331	else
1332	{
1333	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1334	* not be executed. See hmQueueInvlPage() where it is commented
1335	* out. Support individual entry flushing someday. */
1336	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1337	{
1338	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1340	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1341	}
1342	else
1343	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1344	}
1345
1346	pVCpu->hm.s.TlbShootdown.cPages = 0;
1347	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1348	}
1349
1350
1351	/**
1352	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1353	*
1354	* @returns VBox status code.
1355	* @param pVM Pointer to the VM.
1356	* @param pVCpu Pointer to the VMCPU.
1357	*
1358	* @remarks Called with interrupts disabled.
1359	*/
1360	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1361	{
1362	AssertPtr(pVM);
1363	AssertPtr(pVCpu);
1364	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1365	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1366
1367	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1368
1369	/*
1370	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1371	* This can happen both for start & resume due to long jumps back to ring-3.
1372	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1373	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1374	*/
1375	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1376	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1377	{
1378	pVCpu->hm.s.fForceTLBFlush = true;
1379	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1380	}
1381
1382	/* Check for explicit TLB shootdown flushes. */
1383	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1384	{
1385	/*
1386	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1387	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1388	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1389	*/
1390	pVCpu->hm.s.fForceTLBFlush = true;
1391	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1392	}
1393
1394	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1395	if (pVCpu->hm.s.fForceTLBFlush)
1396	{
1397	++pCpu->uCurrentAsid;
1398	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1399	{
1400	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1401	pCpu->fFlushAsidBeforeUse = true;
1402	pCpu->cTlbFlushes++;
1403	}
1404
1405	pVCpu->hm.s.fForceTLBFlush = false;
1406	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1407	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1408	if (pCpu->fFlushAsidBeforeUse)
1409	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1410	}
1411	else
1412	{
1413	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1414	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1415	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1416	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1417
1418	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1419	* not be executed. See hmQueueInvlPage() where it is commented
1420	* out. Support individual entry flushing someday. */
1421	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1422	{
1423	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1424	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1425	{
1426	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1427	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1428	}
1429	else
1430	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1431	}
1432	else
1433	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1434	}
1435
1436	pVCpu->hm.s.TlbShootdown.cPages = 0;
1437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1438
1439	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1440	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1441	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1442	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1443	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1444	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1445
1446	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1447	AssertRC(rc);
1448	}
1449
1450
1451	/**
1452	* Flushes the guest TLB entry based on CPU capabilities.
1453	*
1454	* @param pVCpu Pointer to the VMCPU.
1455	*/
1456	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1457	{
1458	PVM pVM = pVCpu->CTX_SUFF(pVM);
1459	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1460	{
1461	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1462	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1463	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1464	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1465	default:
1466	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1467	break;
1468	}
1469	}
1470
1471
1472	/**
1473	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1474	* TLB entries from the host TLB before VM-entry.
1475	*
1476	* @returns VBox status code.
1477	* @param pVM Pointer to the VM.
1478	*/
1479	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1480	{
1481	/*
1482	* Determine optimal flush type for nested paging.
1483	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1484	* guest execution (see hmR3InitFinalizeR0()).
1485	*/
1486	if (pVM->hm.s.fNestedPaging)
1487	{
1488	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1489	{
1490	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1491	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1492	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1493	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1494	else
1495	{
1496	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1497	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1498	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1499	}
1500
1501	/* Make sure the write-back cacheable memory type for EPT is supported. */
1502	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1503	{
1504	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1505	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1506	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1507	}
1508	}
1509	else
1510	{
1511	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1512	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1513	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1514	}
1515	}
1516
1517	/*
1518	* Determine optimal flush type for VPID.
1519	*/
1520	if (pVM->hm.s.vmx.fVpid)
1521	{
1522	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1523	{
1524	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1525	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1526	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1527	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1528	else
1529	{
1530	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1531	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1532	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1533	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1534	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1535	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1536	pVM->hm.s.vmx.fVpid = false;
1537	}
1538	}
1539	else
1540	{
1541	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1542	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1543	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1544	pVM->hm.s.vmx.fVpid = false;
1545	}
1546	}
1547
1548	/*
1549	* Setup the handler for flushing tagged-TLBs.
1550	*/
1551	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1552	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1553	else if (pVM->hm.s.fNestedPaging)
1554	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1555	else if (pVM->hm.s.vmx.fVpid)
1556	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1557	else
1558	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1559	return VINF_SUCCESS;
1560	}
1561
1562
1563	/**
1564	* Sets up pin-based VM-execution controls in the VMCS.
1565	*
1566	* @returns VBox status code.
1567	* @param pVM Pointer to the VM.
1568	* @param pVCpu Pointer to the VMCPU.
1569	*/
1570	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1571	{
1572	AssertPtr(pVM);
1573	AssertPtr(pVCpu);
1574
1575	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1576	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1577
1578	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1579	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1580	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1581
1582	/* Enable the VMX preemption timer. */
1583	if (pVM->hm.s.vmx.fUsePreemptTimer)
1584	{
1585	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1586	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1587	}
1588
1589	if ((val & zap) != val)
1590	{
1591	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1592	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1593	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1594	}
1595
1596	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1597	AssertRCReturn(rc, rc);
1598
1599	/* Update VCPU with the currently set pin-based VM-execution controls. */
1600	pVCpu->hm.s.vmx.u32PinCtls = val;
1601	return rc;
1602	}
1603
1604
1605	/**
1606	* Sets up processor-based VM-execution controls in the VMCS.
1607	*
1608	* @returns VBox status code.
1609	* @param pVM Pointer to the VM.
1610	* @param pVMCPU Pointer to the VMCPU.
1611	*/
1612	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1613	{
1614	AssertPtr(pVM);
1615	AssertPtr(pVCpu);
1616
1617	int rc = VERR_INTERNAL_ERROR_5;
1618	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1619	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1620
1621	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1622	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1623	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1624	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1625	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1626	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1627	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1628
1629	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1630	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1631	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1632	{
1633	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1634	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1635	}
1636
1637	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1638	if (!pVM->hm.s.fNestedPaging)
1639	{
1640	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1641	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1644	}
1645
1646	/* Use TPR shadowing if supported by the CPU. */
1647	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1648	{
1649	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1650	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1651	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1652	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1653	AssertRCReturn(rc, rc);
1654
1655	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1656	/* CR8 writes causes a VM-exit based on TPR threshold. */
1657	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1658	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1659	}
1660	else
1661	{
1662	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1663	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1664	}
1665
1666	/* Use MSR-bitmaps if supported by the CPU. */
1667	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1668	{
1669	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1670
1671	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1672	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1673	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1674	AssertRCReturn(rc, rc);
1675
1676	/*
1677	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1678	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1679	*/
1680	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1681	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1682	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1683	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1684	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1685	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1686	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1687	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1688	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1689	}
1690
1691	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1692	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1693	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1694
1695	if ((val & zap) != val)
1696	{
1697	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1698	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1699	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1700	}
1701
1702	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1703	AssertRCReturn(rc, rc);
1704
1705	/* Update VCPU with the currently set processor-based VM-execution controls. */
1706	pVCpu->hm.s.vmx.u32ProcCtls = val;
1707
1708	/*
1709	* Secondary processor-based VM-execution controls.
1710	*/
1711	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1712	{
1713	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1714	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1715
1716	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1717	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1718
1719	if (pVM->hm.s.fNestedPaging)
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1721	else
1722	{
1723	/*
1724	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1725	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1726	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1727	*/
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1729	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1730	}
1731
1732	if (pVM->hm.s.vmx.fVpid)
1733	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1734
1735	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1736	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1737
1738	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1739	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1740	* done dynamically. */
1741	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1742	{
1743	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1744	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1746	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1747	AssertRCReturn(rc, rc);
1748	}
1749
1750	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1751	{
1752	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1753	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1754	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1755	}
1756
1757	if ((val & zap) != val)
1758	{
1759	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1760	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1761	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1762	}
1763
1764	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1765	AssertRCReturn(rc, rc);
1766
1767	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1768	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1769	}
1770
1771	return VINF_SUCCESS;
1772	}
1773
1774
1775	/**
1776	* Sets up miscellaneous (everything other than Pin & Processor-based
1777	* VM-execution) control fields in the VMCS.
1778	*
1779	* @returns VBox status code.
1780	* @param pVM Pointer to the VM.
1781	* @param pVCpu Pointer to the VMCPU.
1782	*/
1783	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1784	{
1785	AssertPtr(pVM);
1786	AssertPtr(pVCpu);
1787
1788	int rc = VERR_GENERAL_FAILURE;
1789
1790	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1791	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1792
1793	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1794
1795	/*
1796	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1797	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1798	* We thus use the exception bitmap to control it rather than use both.
1799	*/
1800	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1801	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1802
1803	/** @todo Explore possibility of using IO-bitmaps. */
1804	/* All IO & IOIO instructions cause VM-exits. */
1805	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1806	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1807
1808	/* Setup MSR autoloading/autostoring. */
1809	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1810	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1811	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1812	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1813	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1814	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1815
1816	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1817	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1818	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1820
1821	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1823
1824	/* Setup debug controls */
1825	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1826	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1827	AssertRCReturn(rc, rc);
1828	return rc;
1829	}
1830
1831
1832	/**
1833	* Sets up the initial exception bitmap in the VMCS based on static conditions
1834	* (i.e. conditions that cannot ever change at runtime).
1835	*
1836	* @returns VBox status code.
1837	* @param pVM Pointer to the VM.
1838	* @param pVCpu Pointer to the VMCPU.
1839	*/
1840	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1841	{
1842	AssertPtr(pVM);
1843	AssertPtr(pVCpu);
1844
1845	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1846
1847	uint32_t u32XcptBitmap = 0;
1848
1849	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1850	if (!pVM->hm.s.fNestedPaging)
1851	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1852
1853	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1854	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1855	AssertRCReturn(rc, rc);
1856	return rc;
1857	}
1858
1859
1860	/**
1861	* Sets up the initial guest-state mask. The guest-state mask is consulted
1862	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1863	* for the nested virtualization case (as it would cause a VM-exit).
1864	*
1865	* @param pVCpu Pointer to the VMCPU.
1866	*/
1867	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1868	{
1869	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1870	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1871	return VINF_SUCCESS;
1872	}
1873
1874
1875	/**
1876	* Does per-VM VT-x initialization.
1877	*
1878	* @returns VBox status code.
1879	* @param pVM Pointer to the VM.
1880	*/
1881	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1882	{
1883	LogFlowFunc(("pVM=%p\n", pVM));
1884
1885	int rc = hmR0VmxStructsAlloc(pVM);
1886	if (RT_FAILURE(rc))
1887	{
1888	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1889	return rc;
1890	}
1891
1892	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1893	{
1894	PVMCPU pVCpu = &pVM->aCpus[i];
1895
1896	/* Current guest paging mode. */
1897	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1898	}
1899
1900	return VINF_SUCCESS;
1901	}
1902
1903
1904	/**
1905	* Does per-VM VT-x termination.
1906	*
1907	* @returns VBox status code.
1908	* @param pVM Pointer to the VM.
1909	*/
1910	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1911	{
1912	LogFlowFunc(("pVM=%p\n", pVM));
1913
1914	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1915	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1916	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1917	#endif
1918	hmR0VmxStructsFree(pVM);
1919	return VINF_SUCCESS;
1920	}
1921
1922
1923	/**
1924	* Sets up the VM for execution under VT-x.
1925	* This function is only called once per-VM during initalization.
1926	*
1927	* @returns VBox status code.
1928	* @param pVM Pointer to the VM.
1929	*/
1930	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1931	{
1932	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1933	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1934
1935	LogFlowFunc(("pVM=%p\n", pVM));
1936
1937	/*
1938	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1939	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1940	*/
1941	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1942	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1943	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1944	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1945	{
1946	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1947	return VERR_INTERNAL_ERROR;
1948	}
1949
1950	/* Initialize these always, see hmR3InitFinalizeR0().*/
1951	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1952	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1953
1954	/* Setup the tagged-TLB flush handlers. */
1955	int rc = hmR0VmxSetupTaggedTlb(pVM);
1956	if (RT_FAILURE(rc))
1957	{
1958	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1959	return rc;
1960	}
1961
1962	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1963	{
1964	PVMCPU pVCpu = &pVM->aCpus[i];
1965	AssertPtr(pVCpu);
1966	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1967
1968	/* Set revision dword at the beginning of the VMCS structure. */
1969	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1970
1971	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1972	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1973	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1974	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1975
1976	/* Load this VMCS as the current VMCS. */
1977	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1978	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1979	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1980
1981	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1982	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1983	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1984
1985	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1986	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1987	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1988
1989	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1990	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1991	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1992
1993	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1994	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1995	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1996
1997	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000
2001	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2002	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2003	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2004	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2005	#endif
2006
2007	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2008	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2013	}
2014
2015	return VINF_SUCCESS;
2016	}
2017
2018
2019	/**
2020	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2021	* the VMCS.
2022	*
2023	* @returns VBox status code.
2024	* @param pVM Pointer to the VM.
2025	* @param pVCpu Pointer to the VMCPU.
2026	*/
2027	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2028	{
2029	RTCCUINTREG uReg = ASMGetCR0();
2030	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2031
2032	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2033	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2034	if (HMVMX_IS_64BIT_HOST_MODE())
2035	{
2036	uint64_t uReg = hmR0Get64bitCR3();
2037	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2038	}
2039	else
2040	#endif
2041	{
2042	uReg = ASMGetCR3();
2043	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2044	}
2045
2046	uReg = ASMGetCR4();
2047	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2048	AssertRCReturn(rc, rc);
2049	return rc;
2050	}
2051
2052
2053	/**
2054	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2055	* the host-state area in the VMCS.
2056	*
2057	* @returns VBox status code.
2058	* @param pVM Pointer to the VM.
2059	* @param pVCpu Pointer to the VMCPU.
2060	*/
2061	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2062	{
2063	int rc = VERR_INTERNAL_ERROR_5;
2064	RTSEL uSelCS = 0;
2065	RTSEL uSelSS = 0;
2066	RTSEL uSelDS = 0;
2067	RTSEL uSelES = 0;
2068	RTSEL uSelFS = 0;
2069	RTSEL uSelGS = 0;
2070	RTSEL uSelTR = 0;
2071
2072	/*
2073	* Host Selector registers.
2074	*/
2075	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2076	if (HMVMX_IS_64BIT_HOST_MODE())
2077	{
2078	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2079	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2080	}
2081	else
2082	{
2083	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2084	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2085	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2086	}
2087	#else
2088	uSelCS = ASMGetCS();
2089	uSelSS = ASMGetSS();
2090	#endif
2091
2092	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2093	/** @todo Verify if we have any platform that actually run with DS or ES with
2094	* RPL != 0 in kernel space. */
2095	uSelDS = 0;
2096	uSelES = 0;
2097	uSelFS = 0;
2098	uSelGS = 0;
2099	uSelTR = ASMGetTR();
2100
2101	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2102	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2103	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2104	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2105	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2106	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2107	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2108	Assert(uSelCS != 0);
2109	Assert(uSelTR != 0);
2110
2111	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2112	#if 0
2113	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2114	Assert(uSelSS != 0);
2115	#endif
2116
2117	/* Write these host selector fields into the host-state area in the VMCS. */
2118	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2119	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2120	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2121	#if 0
2122	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2123	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2124	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2125	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2126	#endif
2127	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2128	AssertRCReturn(rc, rc);
2129
2130	/*
2131	* Host GDTR and IDTR.
2132	*/
2133	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2134	* be safe to -not- save and restore GDTR and IDTR in the assembly
2135	* code and just do it here and don't care if the limits are zapped on
2136	* VM-exit. */
2137	RTGDTR Gdtr;
2138	RT_ZERO(Gdtr);
2139	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2140	if (HMVMX_IS_64BIT_HOST_MODE())
2141	{
2142	X86XDTR64 Gtr64;
2143	X86XDTR64 Idtr64;
2144	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2145	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2146	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2147	Gdtr.cbGdt = Gdtr64.cb;
2148	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2149	}
2150	else
2151	#endif
2152	{
2153	RTIDTR Idtr;
2154	ASMGetGDTR(&Gdtr);
2155	ASMGetIDTR(&Idtr);
2156	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2157	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2158	}
2159	AssertRCReturn(rc, rc);
2160
2161	/*
2162	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2163	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2164	*/
2165	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2166	{
2167	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2168	return VERR_VMX_INVALID_HOST_STATE;
2169	}
2170
2171	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2172	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2173	if (HMVMX_IS_64BIT_HOST_MODE())
2174	{
2175	/* We need the 64-bit TR base for hybrid darwin. */
2176	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2177	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2178	}
2179	else
2180	#endif
2181	{
2182	uintptr_t uTRBase;
2183	#if HC_ARCH_BITS == 64
2184	uTRBase = X86DESC64_BASE(pDesc);
2185	#else
2186	uTRBase = X86DESC_BASE(pDesc);
2187	#endif
2188	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2189	}
2190	AssertRCReturn(rc, rc);
2191
2192	/*
2193	* Host FS base and GS base.
2194	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2195	* would take care of the bases. In 64-bit, the MSRs come into play.
2196	*/
2197	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2198	if (HMVMX_IS_64BIT_HOST_MODE())
2199	{
2200	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2201	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2202	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2203	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2204	AssertRCReturn(rc, rc);
2205	}
2206	#endif
2207	return rc;
2208	}
2209
2210
2211	/**
2212	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2213	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2214	* the host after every successful VM exit.
2215	*
2216	* @returns VBox status code.
2217	* @param pVM Pointer to the VM.
2218	* @param pVCpu Pointer to the VMCPU.
2219	*/
2220	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2221	{
2222	AssertPtr(pVCpu);
2223	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2224
2225	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2226	uint32_t cHostMsrs = 0;
2227	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2228
2229	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2230	{
2231	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2232	pHostMsr->u32Reserved = 0;
2233	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2234	if (CPUMIsGuestInLongMode(pVCpu))
2235	{
2236	/* Must match the EFER value in our 64 bits switcher. */
2237	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2238	}
2239	else
2240	#endif
2241	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2242	pHostMsr++; cHostMsrs++;
2243	}
2244
2245	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2246	if (HMVMX_IS_64BIT_HOST_MODE())
2247	{
2248	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2249	pHostMsr->u32Reserved = 0;
2250	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2251	pHostMsr++; cHostMsrs++;
2252	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2253	pHostMsr->u32Reserved = 0;
2254	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2255	pHostMsr++; cHostMsrs++;
2256	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2257	pHostMsr->u32Reserved = 0;
2258	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2259	pHostMsr++; cHostMsrs++;
2260	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2261	#if 0
2262	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2263	pMsr->u32Reserved = 0;
2264	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2265	pHostMsr++; cHostMsrs++;
2266	#endif
2267	}
2268	#endif
2269
2270	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2271	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2272	{
2273	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2274	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2275	}
2276
2277	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2278
2279	/*
2280	* Host Sysenter MSRs.
2281	*/
2282	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2283	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2284	if (HMVMX_IS_64BIT_HOST_MODE())
2285	{
2286	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2287	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2288	}
2289	else
2290	{
2291	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2292	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2293	}
2294	#elif HC_ARCH_BITS == 32
2295	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2296	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2297	#else
2298	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2299	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2300	#endif
2301	AssertRCReturn(rc, rc);
2302
2303	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2304	* hmR0VmxSetupExitCtls() !! */
2305	return rc;
2306	}
2307
2308
2309	/**
2310	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2311	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2312	* controls".
2313	*
2314	* @returns VBox status code.
2315	* @param pVCpu Pointer to the VMCPU.
2316	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2317	* out-of-sync. Make sure to update the required fields
2318	* before using them.
2319	*
2320	* @remarks No-long-jump zone!!!
2321	*/
2322	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2323	{
2324	int rc = VINF_SUCCESS;
2325	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2326	{
2327	PVM pVM = pVCpu->CTX_SUFF(pVM);
2328	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2329	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2330
2331	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2332	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2333
2334	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2335	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2336	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2337	else
2338	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2339
2340	/*
2341	* The following should not be set (since we're not in SMM mode):
2342	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2343	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2344	*/
2345
2346	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2347	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2348	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2349
2350	if ((val & zap) != val)
2351	{
2352	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2353	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2354	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2355	}
2356
2357	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2358	AssertRCReturn(rc, rc);
2359
2360	/* Update VCPU with the currently set VM-exit controls. */
2361	pVCpu->hm.s.vmx.u32EntryCtls = val;
2362	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2363	}
2364	return rc;
2365	}
2366
2367
2368	/**
2369	* Sets up the VM-exit controls in the VMCS.
2370	*
2371	* @returns VBox status code.
2372	* @param pVM Pointer to the VM.
2373	* @param pVCpu Pointer to the VMCPU.
2374	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2375	* out-of-sync. Make sure to update the required fields
2376	* before using them.
2377	*
2378	* @remarks requires EFER.
2379	*/
2380	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2381	{
2382	int rc = VINF_SUCCESS;
2383	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2384	{
2385	PVM pVM = pVCpu->CTX_SUFF(pVM);
2386	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2387	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2388
2389	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2390	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2391
2392	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2393	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2394	if (HMVMX_IS_64BIT_HOST_MODE())
2395	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2396	else
2397	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2398	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2399	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2400	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2401	else
2402	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2403	#endif
2404
2405	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2406	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2407
2408	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2409	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2410	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2411	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2412	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2413
2414	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2415	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2416
2417	if ((val & zap) != val)
2418	{
2419	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2420	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2421	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2422	}
2423
2424	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2425	AssertRCReturn(rc, rc);
2426
2427	/* Update VCPU with the currently set VM-exit controls. */
2428	pVCpu->hm.s.vmx.u32ExitCtls = val;
2429	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2430	}
2431	return rc;
2432	}
2433
2434
2435	/**
2436	* Loads the guest APIC and related state.
2437	*
2438	* @returns VBox status code.
2439	* @param pVM Pointer to the VM.
2440	* @param pVCpu Pointer to the VMCPU.
2441	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2442	* out-of-sync. Make sure to update the required fields
2443	* before using them.
2444	*/
2445	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2446	{
2447	int rc = VINF_SUCCESS;
2448	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2449	{
2450	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2451	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2452	{
2453	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2454
2455	bool fPendingIntr = false;
2456	uint8_t u8GuestTpr = 0;
2457	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2458	AssertRCReturn(rc, rc);
2459
2460	/*
2461	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2462	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2463	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2464	* the interrupt when we VM-exit for other reasons.
2465	*/
2466	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2467	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2468	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2469	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2470
2471	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2472	AssertRCReturn(rc, rc);
2473
2474	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2475	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2476	{
2477	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2478	pMixedCtx->msrLSTAR = u8GuestTpr;
2479	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2480	{
2481	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2482	if (fPendingIntr)
2483	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2484	else
2485	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2486	}
2487	}
2488	}
2489
2490	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2491	}
2492	return rc;
2493	}
2494
2495
2496	/**
2497	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2498	*
2499	* @returns
2500	* @param pVCpu Pointer to the VMCPU.
2501	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2502	* out-of-sync. Make sure to update the required fields
2503	* before using them.
2504	*
2505	* @remarks No-long-jump zone!!!
2506	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2507	*/
2508	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2509	{
2510	/*
2511	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2512	* inhibit interrupts or clear any existing interrupt-inhibition.
2513	*/
2514	uint32_t uIntrState = 0;
2515	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2516	{
2517	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2518	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2519	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2520	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2521	{
2522	/*
2523	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2524	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2525	*/
2526	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2527	}
2528	else if (pMixedCtx->eflags.Bits.u1IF)
2529	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2530	else
2531	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2532	}
2533	return uIntrState;
2534	}
2535
2536
2537	/**
2538	* Loads the guest's interruptibility-state into the guest-state area in the
2539	* VMCS.
2540	*
2541	* @returns VBox status code.
2542	* @param pVCpu Pointer to the VMCPU.
2543	* @param uIntrState The interruptibility-state to set.
2544	*/
2545	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2546	{
2547	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2548	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2549	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2550	AssertRCReturn(rc, rc);
2551	return rc;
2552	}
2553
2554
2555	/**
2556	* Loads the guest's RIP into the guest-state area in the VMCS.
2557	*
2558	* @returns VBox status code.
2559	* @param pVCpu Pointer to the VMCPU.
2560	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2561	* out-of-sync. Make sure to update the required fields
2562	* before using them.
2563	*
2564	* @remarks No-long-jump zone!!!
2565	*/
2566	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2567	{
2568	int rc = VINF_SUCCESS;
2569	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2570	{
2571	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2572	AssertRCReturn(rc, rc);
2573	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2574	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2575	}
2576	return rc;
2577	}
2578
2579
2580	/**
2581	* Loads the guest's RSP into the guest-state area in the VMCS.
2582	*
2583	* @returns VBox status code.
2584	* @param pVCpu Pointer to the VMCPU.
2585	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2586	* out-of-sync. Make sure to update the required fields
2587	* before using them.
2588	*
2589	* @remarks No-long-jump zone!!!
2590	*/
2591	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2592	{
2593	int rc = VINF_SUCCESS;
2594	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2595	{
2596	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2597	AssertRCReturn(rc, rc);
2598	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2599	}
2600	return rc;
2601	}
2602
2603
2604	/**
2605	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2606	*
2607	* @returns VBox status code.
2608	* @param pVCpu Pointer to the VMCPU.
2609	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2610	* out-of-sync. Make sure to update the required fields
2611	* before using them.
2612	*
2613	* @remarks No-long-jump zone!!!
2614	*/
2615	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2616	{
2617	int rc = VINF_SUCCESS;
2618	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2619	{
2620	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2621	Let us assert it as such and use 32-bit VMWRITE. */
2622	Assert(!(pMixedCtx->rflags.u64 >> 32));
2623	X86EFLAGS uEFlags = pMixedCtx->eflags;
2624	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2625	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2626
2627	/*
2628	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2629	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2630	*/
2631	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2632	{
2633	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2634	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2635	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2636	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2637	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2638	}
2639
2640	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2641	AssertRCReturn(rc, rc);
2642
2643	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2644	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2645	}
2646	return rc;
2647	}
2648
2649
2650	/**
2651	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2652	*
2653	* @returns VBox status code.
2654	* @param pVCpu Pointer to the VMCPU.
2655	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2656	* out-of-sync. Make sure to update the required fields
2657	* before using them.
2658	*
2659	* @remarks No-long-jump zone!!!
2660	*/
2661	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2662	{
2663	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2664	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2665	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2666	return rc;
2667	}
2668
2669
2670	/**
2671	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2672	* in the VMCS.
2673	*
2674	* @returns VBox status code.
2675	* @param pVM Pointer to the VM.
2676	* @param pVCpu Pointer to the VMCPU.
2677	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2678	* out-of-sync. Make sure to update the required fields
2679	* before using them.
2680	*
2681	* @remarks No-long-jump zone!!!
2682	*/
2683	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2684	{
2685	int rc = VINF_SUCCESS;
2686	PVM pVM = pVCpu->CTX_SUFF(pVM);
2687
2688	/*
2689	* Guest CR0.
2690	* Guest FPU.
2691	*/
2692	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2693	{
2694	Assert(!(pCtx->cr0 >> 32));
2695	uint32_t u32GuestCR0 = pCtx->cr0;
2696
2697	/* The guest's view (read access) of its CR0 is unblemished. */
2698	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2699	AssertRCReturn(rc, rc);
2700	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2701
2702	/* Setup VT-x's view of the guest CR0. */
2703	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2704	if (pVM->hm.s.fNestedPaging)
2705	{
2706	if (CPUMIsGuestPagingEnabledEx(pCtx))
2707	{
2708	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2709	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2710	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2711	}
2712	else
2713	{
2714	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2715	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2716	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2717	}
2718
2719	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2720	AssertRCReturn(rc, rc);
2721	}
2722	else
2723	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2724
2725	/*
2726	* Guest FPU bits.
2727	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2728	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2729	*/
2730	u32GuestCR0 \|= X86_CR0_NE;
2731	bool fInterceptNM = false;
2732	if (CPUMIsGuestFPUStateActive(pVCpu))
2733	{
2734	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2735	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2736	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2737	}
2738	else
2739	{
2740	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2741	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2742	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2743	}
2744
2745	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2746	bool fInterceptMF = false;
2747	if (!(pCtx->cr0 & X86_CR0_NE))
2748	fInterceptMF = true;
2749
2750	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2751	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2752	{
2753	Assert(PDMVmmDevHeapIsEnabled(pVM));
2754	Assert(pVM->hm.s.vmx.pRealModeTSS);
2755	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2756	fInterceptNM = true;
2757	fInterceptMF = true;
2758	}
2759	else
2760	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2761
2762	if (fInterceptNM)
2763	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2764	else
2765	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2766
2767	if (fInterceptMF)
2768	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2769	else
2770	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2771
2772	/* Additional intercepts for debugging, define these yourself explicitly. */
2773	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2775	\| RT_BIT(X86_XCPT_DB)
2776	\| RT_BIT(X86_XCPT_DE)
2777	\| RT_BIT(X86_XCPT_NM)
2778	\| RT_BIT(X86_XCPT_UD)
2779	\| RT_BIT(X86_XCPT_NP)
2780	\| RT_BIT(X86_XCPT_SS)
2781	\| RT_BIT(X86_XCPT_GP)
2782	\| RT_BIT(X86_XCPT_PF)
2783	\| RT_BIT(X86_XCPT_MF);
2784	#elif defined(VBOX_ALWAYS_TRAP_PF)
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2786	#endif
2787
2788	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2789
2790	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2791	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2792	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2793	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2794	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2795	else
2796	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2797
2798	u32GuestCR0 \|= uSetCR0;
2799	u32GuestCR0 &= uZapCR0;
2800	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2801
2802	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2803	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2804	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2805	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2806
2807	/*
2808	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2809	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2810	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2811	*/
2812	uint64_t u64CR0Mask = 0;
2813	u64CR0Mask = X86_CR0_PE
2814	\| X86_CR0_NE
2815	\| X86_CR0_WP
2816	\| X86_CR0_PG
2817	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2818	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2819	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2820	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2821	u64CR0Mask &= ~X86_CR0_PE;
2822	if (pVM->hm.s.fNestedPaging)
2823	u64CR0Mask &= ~X86_CR0_WP;
2824
2825	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2826	if (fInterceptNM)
2827	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2828	else
2829	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2830
2831	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2832	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2833	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2834	AssertRCReturn(rc, rc);
2835
2836	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2837	}
2838
2839	/*
2840	* Guest CR2.
2841	* It's always loaded in the assembler code. Nothing to do here.
2842	*/
2843
2844	/*
2845	* Guest CR3.
2846	*/
2847	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2848	{
2849	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2850	if (pVM->hm.s.fNestedPaging)
2851	{
2852	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2853
2854	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2855	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2856	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2857	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2858
2859	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2860	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2861	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2862
2863	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2864	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2865	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2866	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2867
2868	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2869	AssertRCReturn(rc, rc);
2870	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2871
2872	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2873	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2874	{
2875	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2876	if (CPUMIsGuestInPAEModeEx(pCtx))
2877	{
2878	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2879	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2880	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2881	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2882	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2883	AssertRCReturn(rc, rc);
2884	}
2885
2886	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2887	have Unrestricted Execution to handle the guest when it's not using paging. */
2888	GCPhysGuestCR3 = pCtx->cr3;
2889	}
2890	else
2891	{
2892	/*
2893	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2894	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2895	* EPT takes care of translating it to host-physical addresses.
2896	*/
2897	RTGCPHYS GCPhys;
2898	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2899	Assert(PDMVmmDevHeapIsEnabled(pVM));
2900
2901	/* We obtain it here every time as the guest could have relocated this PCI region. */
2902	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2903	AssertRCReturn(rc, rc);
2904
2905	GCPhysGuestCR3 = GCPhys;
2906	}
2907	}
2908	else
2909	{
2910	/* Non-nested paging case, just use the hypervisor's CR3. */
2911	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2912	}
2913
2914	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2915	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2916	AssertRCReturn(rc, rc);
2917
2918	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2919	}
2920
2921	/*
2922	* Guest CR4.
2923	*/
2924	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2925	{
2926	Assert(!(pCtx->cr4 >> 32));
2927	uint32_t u32GuestCR4 = pCtx->cr4;
2928
2929	/* The guest's view of its CR4 is unblemished. */
2930	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2931	AssertRCReturn(rc, rc);
2932	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2933
2934	/* Setup VT-x's view of the guest CR4. */
2935	/*
2936	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2937	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2938	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2939	*/
2940	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2941	{
2942	Assert(pVM->hm.s.vmx.pRealModeTSS);
2943	Assert(PDMVmmDevHeapIsEnabled(pVM));
2944	u32GuestCR4 &= ~X86_CR4_VME;
2945	}
2946
2947	if (pVM->hm.s.fNestedPaging)
2948	{
2949	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2950	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2951	{
2952	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2953	u32GuestCR4 \|= X86_CR4_PSE;
2954	/* Our identity mapping is a 32 bits page directory. */
2955	u32GuestCR4 &= ~X86_CR4_PAE;
2956	}
2957	/* else use guest CR4.*/
2958	}
2959	else
2960	{
2961	/*
2962	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2963	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2964	*/
2965	switch (pVCpu->hm.s.enmShadowMode)
2966	{
2967	case PGMMODE_REAL: /* Real-mode. */
2968	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2969	case PGMMODE_32_BIT: /* 32-bit paging. */
2970	{
2971	u32GuestCR4 &= ~X86_CR4_PAE;
2972	break;
2973	}
2974
2975	case PGMMODE_PAE: /* PAE paging. */
2976	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2977	{
2978	u32GuestCR4 \|= X86_CR4_PAE;
2979	break;
2980	}
2981
2982	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2983	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2984	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2985	break;
2986	#endif
2987	default:
2988	AssertFailed();
2989	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2990	}
2991	}
2992
2993	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2994	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2995	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2996	u32GuestCR4 \|= uSetCR4;
2997	u32GuestCR4 &= uZapCR4;
2998
2999	/* Write VT-x's view of the guest CR4 into the VMCS. */
3000	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3001	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3002
3003	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3004	uint64_t u64CR4Mask = 0;
3005	u64CR4Mask = X86_CR4_VME
3006	\| X86_CR4_PAE
3007	\| X86_CR4_PGE
3008	\| X86_CR4_PSE
3009	\| X86_CR4_VMXE;
3010	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3011	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3012	AssertRCReturn(rc, rc);
3013
3014	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3015	}
3016	return rc;
3017	}
3018
3019
3020	/**
3021	* Loads the guest debug registers into the guest-state area in the VMCS.
3022	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3023	*
3024	* @returns VBox status code.
3025	* @param pVCpu Pointer to the VMCPU.
3026	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3027	* out-of-sync. Make sure to update the required fields
3028	* before using them.
3029	*
3030	* @remarks No-long-jump zone!!!
3031	*/
3032	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3033	{
3034	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3035	return VINF_SUCCESS;
3036
3037	#ifdef VBOX_STRICT
3038	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3039	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3040	{
3041	Assert((pMixedCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3042	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3043	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3044	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3045	}
3046	#endif
3047
3048	int rc = VERR_INTERNAL_ERROR_5;
3049	PVM pVM = pVCpu->CTX_SUFF(pVM);
3050	bool fInterceptDB = false;
3051	bool fInterceptMovDRx = false;
3052	if (DBGFIsStepping(pVCpu))
3053	{
3054	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3055	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3056	{
3057	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3058	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3059	AssertRCReturn(rc, rc);
3060	Assert(fInterceptDB == false);
3061	}
3062	else
3063	fInterceptDB = true;
3064	}
3065
3066	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3067	{
3068	if (!CPUMIsHyperDebugStateActive(pVCpu))
3069	{
3070	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3071	AssertRC(rc);
3072	}
3073	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3074	fInterceptMovDRx = true;
3075	}
3076	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3077	{
3078	if (!CPUMIsGuestDebugStateActive(pVCpu))
3079	{
3080	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3081	AssertRC(rc);
3082	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3083	}
3084	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3085	Assert(fInterceptMovDRx == false);
3086	}
3087	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3088	{
3089	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3090	fInterceptMovDRx = true;
3091	}
3092
3093	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3094	if (fInterceptDB)
3095	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3096	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3097	{
3098	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3099	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3100	#endif
3101	}
3102
3103	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3104	if (fInterceptMovDRx)
3105	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3106	else
3107	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3108
3109	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3110	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3111
3112	/* The guest's view of its DR7 is unblemished. */
3113	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3114
3115	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3116	return rc;
3117	}
3118
3119
3120	#ifdef VBOX_STRICT
3121	/**
3122	* Strict function to validate segment registers.
3123	*/
3124	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3125	{
3126	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3127	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3128	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3129	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3130	{
3131	/* Protected mode checks */
3132	/* CS */
3133	Assert(pCtx->cs.Attr.n.u1Present);
3134	Assert(!(pCtx->cs.Attr.u & 0xf00));
3135	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3136	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3137	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3138	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3139	\|\| (pCtx->cs.Attr.n.u1Granularity));
3140	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3141	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3142	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3143	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3144	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3145	else
3146	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3147	/* SS */
3148	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3149	{
3150	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3151	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3152	Assert(pCtx->ss.Attr.n.u1Present);
3153	Assert(!(pCtx->ss.Attr.u & 0xf00));
3154	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3155	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3156	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3157	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3158	\|\| (pCtx->ss.Attr.n.u1Granularity));
3159	}
3160	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3161	/* CR0 might not be up-to-date here always, hence disabled. */
3162	#if 0
3163	if (!pCtx->cr0 & X86_CR0_PE)
3164	Assert(!pCtx->ss.Attr.n.u2Dpl);
3165	#endif
3166	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3167	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3168	{
3169	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3170	Assert(pCtx->ds.Attr.n.u1Present);
3171	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3172	Assert(!(pCtx->ds.Attr.u & 0xf00));
3173	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3174	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3175	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3176	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3177	\|\| (pCtx->ds.Attr.n.u1Granularity));
3178	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3179	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3180	}
3181	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3182	{
3183	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3184	Assert(pCtx->es.Attr.n.u1Present);
3185	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3186	Assert(!(pCtx->es.Attr.u & 0xf00));
3187	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3188	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3189	\|\| !(pCtx->es.Attr.n.u1Granularity));
3190	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3191	\|\| (pCtx->es.Attr.n.u1Granularity));
3192	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3193	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3194	}
3195	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3196	{
3197	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3198	Assert(pCtx->fs.Attr.n.u1Present);
3199	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3200	Assert(!(pCtx->fs.Attr.u & 0xf00));
3201	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3202	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3203	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3204	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3205	\|\| (pCtx->fs.Attr.n.u1Granularity));
3206	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3207	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3208	}
3209	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3210	{
3211	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3212	Assert(pCtx->gs.Attr.n.u1Present);
3213	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3214	Assert(!(pCtx->gs.Attr.u & 0xf00));
3215	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3216	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3217	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3218	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3219	\|\| (pCtx->gs.Attr.n.u1Granularity));
3220	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3221	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3222	}
3223	/* 64-bit capable CPUs. */
3224	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3225	Assert(!(pCtx->cs.u64Base >> 32));
3226	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3227	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3228	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3229	# endif
3230	}
3231	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3232	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3233	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3234	{
3235	/* Real and v86 mode checks. */
3236	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3237	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3238	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3239	{
3240	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3241	}
3242	else
3243	{
3244	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3245	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3246	}
3247
3248	/* CS */
3249	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3250	Assert(pCtx->cs.u32Limit == 0xffff);
3251	Assert(u32CSAttr == 0xf3);
3252	/* SS */
3253	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3254	Assert(pCtx->ss.u32Limit == 0xffff);
3255	Assert(u32SSAttr == 0xf3);
3256	/* DS */
3257	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3258	Assert(pCtx->ds.u32Limit == 0xffff);
3259	Assert(u32DSAttr == 0xf3);
3260	/* ES */
3261	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3262	Assert(pCtx->es.u32Limit == 0xffff);
3263	Assert(u32ESAttr == 0xf3);
3264	/* FS */
3265	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3266	Assert(pCtx->fs.u32Limit == 0xffff);
3267	Assert(u32FSAttr == 0xf3);
3268	/* GS */
3269	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3270	Assert(pCtx->gs.u32Limit == 0xffff);
3271	Assert(u32GSAttr == 0xf3);
3272	/* 64-bit capable CPUs. */
3273	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3274	Assert(!(pCtx->cs.u64Base >> 32));
3275	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3276	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3277	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3278	# endif
3279	}
3280	}
3281	#endif /* VBOX_STRICT */
3282
3283
3284	/**
3285	* Writes a guest segment register into the guest-state area in the VMCS.
3286	*
3287	* @returns VBox status code.
3288	* @param pVCpu Pointer to the VMCPU.
3289	* @param idxSel Index of the selector in the VMCS.
3290	* @param idxLimit Index of the segment limit in the VMCS.
3291	* @param idxBase Index of the segment base in the VMCS.
3292	* @param idxAccess Index of the access rights of the segment in the VMCS.
3293	* @param pSelReg Pointer to the segment selector.
3294	* @param pCtx Pointer to the guest-CPU context.
3295	*
3296	* @remarks No-long-jump zone!!!
3297	*/
3298	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3299	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3300	{
3301	int rc;
3302	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3303	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3304	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3305	AssertRCReturn(rc, rc);
3306
3307	uint32_t u32Access = pSelReg->Attr.u;
3308	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3309	{
3310	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3311	u32Access = 0xf3;
3312	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3313	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3314	}
3315	else
3316	{
3317	/*
3318	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3319	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3320	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3321	* loaded in protected-mode have their attribute as 0.
3322	*/
3323	if (!u32Access)
3324	u32Access = HMVMX_SEL_UNUSABLE;
3325	}
3326
3327	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3328	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3329	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3330
3331	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3332	AssertRCReturn(rc, rc);
3333	return rc;
3334	}
3335
3336
3337	/**
3338	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3339	* into the guest-state area in the VMCS.
3340	*
3341	* @returns VBox status code.
3342	* @param pVM Pointer to the VM.
3343	* @param pVCPU Pointer to the VMCPU.
3344	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3345	* out-of-sync. Make sure to update the required fields
3346	* before using them.
3347	*
3348	* @remarks No-long-jump zone!!!
3349	*/
3350	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3351	{
3352	int rc = VERR_INTERNAL_ERROR_5;
3353	PVM pVM = pVCpu->CTX_SUFF(pVM);
3354
3355	/*
3356	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3357	*/
3358	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3359	{
3360	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3361	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3362	{
3363	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3364	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3365	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3366	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3367	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3368	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3369	}
3370
3371	#ifdef VBOX_WITH_REM
3372	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3373	{
3374	Assert(pVM->hm.s.vmx.pRealModeTSS);
3375	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3376	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3377	{
3378	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3379	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3380	&& enmGuestMode >= PGMMODE_PROTECTED)
3381	{
3382	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3383	in real-mode (e.g. OpenBSD 4.0) */
3384	REMFlushTBs(pVM);
3385	Log(("Load: Switch to protected mode detected!\n"));
3386	}
3387	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3388	}
3389	}
3390	#endif
3391	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3392	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3393	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3394	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3395	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3396	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3397	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3398	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3399	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3400	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3401	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3402	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3403	AssertRCReturn(rc, rc);
3404
3405	#ifdef VBOX_STRICT
3406	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3407	#endif
3408	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3409	}
3410
3411	/*
3412	* Guest TR.
3413	*/
3414	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3415	{
3416	/*
3417	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3418	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3419	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3420	*/
3421	uint16_t u16Sel = 0;
3422	uint32_t u32Limit = 0;
3423	uint64_t u64Base = 0;
3424	uint32_t u32AccessRights = 0;
3425
3426	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3427	{
3428	u16Sel = pMixedCtx->tr.Sel;
3429	u32Limit = pMixedCtx->tr.u32Limit;
3430	u64Base = pMixedCtx->tr.u64Base;
3431	u32AccessRights = pMixedCtx->tr.Attr.u;
3432	}
3433	else
3434	{
3435	Assert(pVM->hm.s.vmx.pRealModeTSS);
3436	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3437
3438	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3439	RTGCPHYS GCPhys;
3440	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3441	AssertRCReturn(rc, rc);
3442
3443	X86DESCATTR DescAttr;
3444	DescAttr.u = 0;
3445	DescAttr.n.u1Present = 1;
3446	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3447
3448	u16Sel = 0;
3449	u32Limit = HM_VTX_TSS_SIZE;
3450	u64Base = GCPhys; /* in real-mode phys = virt. */
3451	u32AccessRights = DescAttr.u;
3452	}
3453
3454	/* Validate. */
3455	Assert(!(u16Sel & RT_BIT(2)));
3456	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3457	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3458	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3459	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3460	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3461	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3462	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3463	Assert( (u32Limit & 0xfff) == 0xfff
3464	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3465	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3466	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3467
3468	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3469	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3470	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3471	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3472	AssertRCReturn(rc, rc);
3473
3474	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3475	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3476	}
3477
3478	/*
3479	* Guest GDTR.
3480	*/
3481	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3482	{
3483	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3484	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3485	AssertRCReturn(rc, rc);
3486
3487	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3488	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3489	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3490	}
3491
3492	/*
3493	* Guest LDTR.
3494	*/
3495	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3496	{
3497	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3498	uint32_t u32Access = 0;
3499	if (!pMixedCtx->ldtr.Attr.u)
3500	u32Access = HMVMX_SEL_UNUSABLE;
3501	else
3502	u32Access = pMixedCtx->ldtr.Attr.u;
3503
3504	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3505	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3506	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3507	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3508	AssertRCReturn(rc, rc);
3509
3510	/* Validate. */
3511	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3512	{
3513	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3514	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3515	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3516	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3517	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3518	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3519	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3520	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3521	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3522	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3523	}
3524
3525	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3526	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3527	}
3528
3529	/*
3530	* Guest IDTR.
3531	*/
3532	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3533	{
3534	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3535	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3536	AssertRCReturn(rc, rc);
3537
3538	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3539	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3540	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3541	}
3542
3543	return VINF_SUCCESS;
3544	}
3545
3546
3547	/**
3548	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3549	* areas. These MSRs will automatically be loaded to the host CPU on every
3550	* successful VM entry and stored from the host CPU on every successful VM exit.
3551	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3552	*
3553	* @returns VBox status code.
3554	* @param pVCpu Pointer to the VMCPU.
3555	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3556	* out-of-sync. Make sure to update the required fields
3557	* before using them.
3558	*
3559	* @remarks No-long-jump zone!!!
3560	*/
3561	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3562	{
3563	AssertPtr(pVCpu);
3564	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3565
3566	/*
3567	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3568	*/
3569	int rc = VINF_SUCCESS;
3570	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3571	{
3572	PVM pVM = pVCpu->CTX_SUFF(pVM);
3573	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3574	uint32_t cGuestMsrs = 0;
3575
3576	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3577	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3578	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3579	if (fSupportsNX \|\| fSupportsLongMode)
3580	{
3581	/** @todo support save IA32_EFER, i.e.
3582	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3583	* guest EFER need not be part of the VM-entry MSR-load area. */
3584	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3585	pGuestMsr->u32Reserved = 0;
3586	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3587	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3588	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3589	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3590	pGuestMsr++; cGuestMsrs++;
3591	if (fSupportsLongMode)
3592	{
3593	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3594	pGuestMsr->u32Reserved = 0;
3595	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3596	pGuestMsr++; cGuestMsrs++;
3597	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3598	pGuestMsr->u32Reserved = 0;
3599	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3600	pGuestMsr++; cGuestMsrs++;
3601	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3602	pGuestMsr->u32Reserved = 0;
3603	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3604	pGuestMsr++; cGuestMsrs++;
3605	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3606	#if 0
3607	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3608	pGuestMsr->u32Reserved = 0;
3609	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3610	pGuestMsr++; cGuestMsrs++;
3611	#endif
3612	}
3613	}
3614
3615	/*
3616	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3617	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3618	*/
3619	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3620	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3621	{
3622	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3623	pGuestMsr->u32Reserved = 0;
3624	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3625	AssertRCReturn(rc, rc);
3626	pGuestMsr++; cGuestMsrs++;
3627	}
3628
3629	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3630	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3631	{
3632	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3633	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3634	}
3635
3636	/* Update the VCPU's copy of the guest MSR count. */
3637	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3638	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3639	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3640	AssertRCReturn(rc, rc);
3641
3642	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3643	}
3644
3645	/*
3646	* Guest Sysenter MSRs.
3647	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3648	* VM-exits on WRMSRs for these MSRs.
3649	*/
3650	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3651	{
3652	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3653	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3654	}
3655	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3656	{
3657	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3658	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3659	}
3660	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3661	{
3662	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3663	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3664	}
3665	AssertRCReturn(rc, rc);
3666
3667	return rc;
3668	}
3669
3670
3671	/**
3672	* Loads the guest activity state into the guest-state area in the VMCS.
3673	*
3674	* @returns VBox status code.
3675	* @param pVCpu Pointer to the VMCPU.
3676	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3677	* out-of-sync. Make sure to update the required fields
3678	* before using them.
3679	*
3680	* @remarks No-long-jump zone!!!
3681	*/
3682	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3683	{
3684	/** @todo See if we can make use of other states, e.g.
3685	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3686	int rc = VINF_SUCCESS;
3687	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3688	{
3689	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3690	AssertRCReturn(rc, rc);
3691	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3692	}
3693	return rc;
3694	}
3695
3696
3697	/**
3698	* Sets up the appropriate function to run guest code.
3699	*
3700	* @returns VBox status code.
3701	* @param pVCpu Pointer to the VMCPU.
3702	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3703	* out-of-sync. Make sure to update the required fields
3704	* before using them.
3705	*
3706	* @remarks No-long-jump zone!!!
3707	*/
3708	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3709	{
3710	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3711	{
3712	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3713	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3714	#endif
3715	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3716	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3717	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3718	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3719	#else
3720	/* 64-bit host or hybrid host. */
3721	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3722	#endif
3723	}
3724	else
3725	{
3726	/* Guest is not in long mode, use the 32-bit handler. */
3727	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3728	}
3729	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3730	return VINF_SUCCESS;
3731	}
3732
3733
3734	/**
3735	* Wrapper for running the guest code in VT-x.
3736	*
3737	* @returns VBox strict status code.
3738	* @param pVM Pointer to the VM.
3739	* @param pVCpu Pointer to the VMCPU.
3740	* @param pCtx Pointer to the guest-CPU context.
3741	*
3742	* @remarks No-long-jump zone!!!
3743	*/
3744	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3745	{
3746	/*
3747	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3748	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3749	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3750	*/
3751	#ifdef VBOX_WITH_KERNEL_USING_XMM
3752	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3753	#else
3754	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3755	#endif
3756	}
3757
3758
3759	/**
3760	* Report world-switch error and dump some useful debug info.
3761	*
3762	* @param pVM Pointer to the VM.
3763	* @param pVCpu Pointer to the VMCPU.
3764	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3765	* @param pCtx Pointer to the guest-CPU context.
3766	* @param pVmxTransient Pointer to the VMX transient structure (only
3767	* exitReason updated).
3768	*/
3769	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3770	{
3771	Assert(pVM);
3772	Assert(pVCpu);
3773	Assert(pCtx);
3774	Assert(pVmxTransient);
3775	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3776
3777	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3778	switch (rcVMRun)
3779	{
3780	case VERR_VMX_INVALID_VMXON_PTR:
3781	AssertFailed();
3782	break;
3783	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3784	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3785	{
3786	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3787	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3788	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3789	AssertRC(rc);
3790
3791	#ifdef VBOX_STRICT
3792	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3793	pVmxTransient->uExitReason));
3794	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3795	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3796	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3797	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3798	else
3799	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3800
3801	/* VMX control bits. */
3802	uint32_t u32Val;
3803	uint64_t u64Val;
3804	RTHCUINTREG uHCReg;
3805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3806	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3808	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3810	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3837	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3838	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3839	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3840	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3841	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3842	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3843	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3844	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3845	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3846	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3847
3848	/* Guest bits. */
3849	RTGCUINTREG uGCReg;
3850	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3851	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3852	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3853	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3854	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3855	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3856	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3857	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3858
3859	/* Host bits. */
3860	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3861	Log(("Host CR0 %#RHr\n", uHCReg));
3862	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3863	Log(("Host CR3 %#RHr\n", uHCReg));
3864	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3865	Log(("Host CR4 %#RHr\n", uHCReg));
3866
3867	RTGDTR HostGdtr;
3868	PCX86DESCHC pDesc;
3869	ASMGetGDTR(&HostGdtr);
3870	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3871	Log(("Host CS %#08x\n", u32Val));
3872	if (u32Val < HostGdtr.cbGdt)
3873	{
3874	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3875	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3876	}
3877
3878	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3879	Log(("Host DS %#08x\n", u32Val));
3880	if (u32Val < HostGdtr.cbGdt)
3881	{
3882	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3883	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3884	}
3885
3886	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3887	Log(("Host ES %#08x\n", u32Val));
3888	if (u32Val < HostGdtr.cbGdt)
3889	{
3890	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3891	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3892	}
3893
3894	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3895	Log(("Host FS %#08x\n", u32Val));
3896	if (u32Val < HostGdtr.cbGdt)
3897	{
3898	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3899	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3900	}
3901
3902	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3903	Log(("Host GS %#08x\n", u32Val));
3904	if (u32Val < HostGdtr.cbGdt)
3905	{
3906	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3907	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3908	}
3909
3910	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3911	Log(("Host SS %#08x\n", u32Val));
3912	if (u32Val < HostGdtr.cbGdt)
3913	{
3914	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3915	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3916	}
3917
3918	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3919	Log(("Host TR %#08x\n", u32Val));
3920	if (u32Val < HostGdtr.cbGdt)
3921	{
3922	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3923	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3924	}
3925
3926	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3927	Log(("Host TR Base %#RHv\n", uHCReg));
3928	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3929	Log(("Host GDTR Base %#RHv\n", uHCReg));
3930	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3931	Log(("Host IDTR Base %#RHv\n", uHCReg));
3932	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3933	Log(("Host SYSENTER CS %#08x\n", u32Val));
3934	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3935	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3936	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3937	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3939	Log(("Host RSP %#RHv\n", uHCReg));
3940	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3941	Log(("Host RIP %#RHv\n", uHCReg));
3942	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3943	if (HMVMX_IS_64BIT_HOST_MODE())
3944	{
3945	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3946	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3947	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3948	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3949	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3950	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3951	}
3952	# endif
3953	#endif /* VBOX_STRICT */
3954	break;
3955	}
3956
3957	default:
3958	/* Impossible */
3959	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3960	break;
3961	}
3962	NOREF(pVM);
3963	}
3964
3965
3966	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3967	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3968	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3969	#endif
3970	#ifdef VBOX_STRICT
3971	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3972	{
3973	switch (idxField)
3974	{
3975	case VMX_VMCS_GUEST_RIP:
3976	case VMX_VMCS_GUEST_RSP:
3977	case VMX_VMCS_GUEST_DR7:
3978	case VMX_VMCS_GUEST_SYSENTER_EIP:
3979	case VMX_VMCS_GUEST_SYSENTER_ESP:
3980	case VMX_VMCS_GUEST_GDTR_BASE:
3981	case VMX_VMCS_GUEST_IDTR_BASE:
3982	case VMX_VMCS_GUEST_CS_BASE:
3983	case VMX_VMCS_GUEST_DS_BASE:
3984	case VMX_VMCS_GUEST_ES_BASE:
3985	case VMX_VMCS_GUEST_FS_BASE:
3986	case VMX_VMCS_GUEST_GS_BASE:
3987	case VMX_VMCS_GUEST_SS_BASE:
3988	case VMX_VMCS_GUEST_LDTR_BASE:
3989	case VMX_VMCS_GUEST_TR_BASE:
3990	case VMX_VMCS_GUEST_CR3:
3991	return true;
3992	}
3993	return false;
3994	}
3995
3996	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3997	{
3998	switch (idxField)
3999	{
4000	/* Read-only fields. */
4001	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4002	return true;
4003	}
4004	/* All readable fields should also be part of the VMCS write cache. */
4005	return hmR0VmxIsValidWriteField(idxField);
4006	}
4007	#endif /* VBOX_STRICT */
4008
4009	/**
4010	* Executes the specified handler in 64-bit mode.
4011	*
4012	* @returns VBox status code.
4013	* @param pVM Pointer to the VM.
4014	* @param pVCpu Pointer to the VMCPU.
4015	* @param pCtx Pointer to the guest CPU context.
4016	* @param pfnHandler Pointer to the RC handler function.
4017	* @param cbParam Number of parameters.
4018	* @param paParam Array of 32-bit parameters.
4019	*/
4020	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4021	uint32_t *paParam)
4022	{
4023	int rc, rc2;
4024	PHMGLOBLCPUINFO pCpu;
4025	RTHCPHYS HCPhysCpuPage;
4026	RTHCUINTREG uOldEFlags;
4027
4028	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4029	Assert(pfnHandler);
4030	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4031	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4032
4033	#ifdef VBOX_STRICT
4034	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4035	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4036
4037	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4038	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4039	#endif
4040
4041	/* Disable interrupts. */
4042	uOldEFlags = ASMIntDisableFlags();
4043
4044	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4045	RTCPUID idHostCpu = RTMpCpuId();
4046	CPUMR0SetLApic(pVM, idHostCpu);
4047	#endif
4048
4049	pCpu = HMR0GetCurrentCpu();
4050	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4051
4052	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4053	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4054
4055	/* Leave VMX Root Mode. */
4056	VMXDisable();
4057
4058	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4059
4060	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4061	CPUMSetHyperEIP(pVCpu, pfnHandler);
4062	for (int i = (int)cbParam - 1; i >= 0; i--)
4063	CPUMPushHyper(pVCpu, paParam[i]);
4064
4065	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4066
4067	/* Call the switcher. */
4068	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4069	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4070
4071	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4072	/* Make sure the VMX instructions don't cause #UD faults. */
4073	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4074
4075	/* Re-enter VMX Root Mode */
4076	rc2 = VMXEnable(HCPhysCpuPage);
4077	if (RT_FAILURE(rc2))
4078	{
4079	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4080	ASMSetFlags(uOldEFlags);
4081	return rc2;
4082	}
4083
4084	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4085	AssertRC(rc2);
4086	Assert(!(ASMGetFlags() & X86_EFL_IF));
4087	ASMSetFlags(uOldEFlags);
4088	return rc;
4089	}
4090
4091
4092	/**
4093	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4094	* supporting 64-bit guests.
4095	*
4096	* @returns VBox status code.
4097	* @param fResume Whether to VMLAUNCH or VMRESUME.
4098	* @param pCtx Pointer to the guest-CPU context.
4099	* @param pCache Pointer to the VMCS cache.
4100	* @param pVM Pointer to the VM.
4101	* @param pVCpu Pointer to the VMCPU.
4102	*/
4103	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4104	{
4105	uint32_t aParam[6];
4106	PHMGLOBLCPUINFO pCpu = NULL;
4107	RTHCPHYS HCPhysCpuPage = 0;
4108	int rc = VERR_INTERNAL_ERROR_5;
4109
4110	pCpu = HMR0GetCurrentCpu();
4111	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4112
4113	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4114	pCache->uPos = 1;
4115	pCache->interPD = PGMGetInterPaeCR3(pVM);
4116	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4117	#endif
4118
4119	#ifdef VBOX_STRICT
4120	pCache->TestIn.HCPhysCpuPage = 0;
4121	pCache->TestIn.HCPhysVmcs = 0;
4122	pCache->TestIn.pCache = 0;
4123	pCache->TestOut.HCPhysVmcs = 0;
4124	pCache->TestOut.pCache = 0;
4125	pCache->TestOut.pCtx = 0;
4126	pCache->TestOut.eflags = 0;
4127	#endif
4128
4129	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4130	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4131	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4132	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4133	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4134	aParam[5] = 0;
4135
4136	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4137	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4138	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4139	#endif
4140	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4141
4142	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4143	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4144	Assert(pCtx->dr[4] == 10);
4145	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4146	#endif
4147
4148	#ifdef VBOX_STRICT
4149	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4150	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4151	pVCpu->hm.s.vmx.HCPhysVmcs));
4152	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4153	pCache->TestOut.HCPhysVmcs));
4154	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4155	pCache->TestOut.pCache));
4156	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4157	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4158	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4159	pCache->TestOut.pCtx));
4160	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4161	#endif
4162	return rc;
4163	}
4164
4165
4166	/**
4167	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4168	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4169	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4170	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4171	*
4172	* @returns VBox status code.
4173	* @param pVM Pointer to the VM.
4174	* @param pVCpu Pointer to the VMCPU.
4175	*/
4176	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4177	{
4178	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4179	{ \
4180	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4181	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4182	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4183	++cReadFields; \
4184	}
4185
4186	AssertPtr(pVM);
4187	AssertPtr(pVCpu);
4188	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4189	uint32_t cReadFields = 0;
4190
4191	/* Guest-natural selector base fields */
4192	#if 0
4193	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4194	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4195	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4196	#endif
4197	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4198	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4199	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4200	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4201	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4202	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4204	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4205	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4208	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4209	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4210	#if 0
4211	/* Unused natural width guest-state fields. */
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4214	#endif
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4217
4218	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4219	#if 0
4220	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4221	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4222	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4223	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4224	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4225	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4226	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4227	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4228	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4229	#endif
4230
4231	/* Natural width guest-state fields. */
4232	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4233	#if 0
4234	/* Currently unused field. */
4235	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4236	#endif
4237
4238	if (pVM->hm.s.fNestedPaging)
4239	{
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4241	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4242	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4243	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4244	}
4245	else
4246	{
4247	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4248	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4249	}
4250
4251	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4252	return VINF_SUCCESS;
4253	}
4254
4255
4256	/**
4257	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4258	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4259	* darwin, running 64-bit guests).
4260	*
4261	* @returns VBox status code.
4262	* @param pVCpu Pointer to the VMCPU.
4263	* @param idxField The VMCS field encoding.
4264	* @param u64Val 16, 32 or 64 bits value.
4265	*/
4266	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4267	{
4268	int rc;
4269	switch (idxField)
4270	{
4271	/*
4272	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4273	*/
4274	/* 64-bit Control fields. */
4275	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4276	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4277	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4278	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4279	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4280	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4281	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4282	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4283	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4284	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4285	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4286	case VMX_VMCS64_CTRL_EPTP_FULL:
4287	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4288	/* 64-bit Guest-state fields. */
4289	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4290	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4291	case VMX_VMCS64_GUEST_PAT_FULL:
4292	case VMX_VMCS64_GUEST_EFER_FULL:
4293	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4294	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4295	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4296	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4297	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4298	/* 64-bit Host-state fields. */
4299	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4300	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4301	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4302	{
4303	rc = VMXWriteVmcs32(idxField, u64Val);
4304	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4305	break;
4306	}
4307
4308	/*
4309	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4310	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4311	*/
4312	/* Natural-width Guest-state fields. */
4313	case VMX_VMCS_GUEST_CR3:
4314	case VMX_VMCS_GUEST_ES_BASE:
4315	case VMX_VMCS_GUEST_CS_BASE:
4316	case VMX_VMCS_GUEST_SS_BASE:
4317	case VMX_VMCS_GUEST_DS_BASE:
4318	case VMX_VMCS_GUEST_FS_BASE:
4319	case VMX_VMCS_GUEST_GS_BASE:
4320	case VMX_VMCS_GUEST_LDTR_BASE:
4321	case VMX_VMCS_GUEST_TR_BASE:
4322	case VMX_VMCS_GUEST_GDTR_BASE:
4323	case VMX_VMCS_GUEST_IDTR_BASE:
4324	case VMX_VMCS_GUEST_DR7:
4325	case VMX_VMCS_GUEST_RSP:
4326	case VMX_VMCS_GUEST_RIP:
4327	case VMX_VMCS_GUEST_SYSENTER_ESP:
4328	case VMX_VMCS_GUEST_SYSENTER_EIP:
4329	{
4330	if (!(u64Val >> 32))
4331	{
4332	/* If this field is 64-bit, VT-x will zero out the top bits. */
4333	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4334	}
4335	else
4336	{
4337	/* Assert that only the 32->64 switcher case should ever come here. */
4338	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4339	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4340	}
4341	break;
4342	}
4343
4344	default:
4345	{
4346	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4347	rc = VERR_INVALID_PARAMETER;
4348	break;
4349	}
4350	}
4351	AssertRCReturn(rc, rc);
4352	return rc;
4353	}
4354
4355
4356	/**
4357	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4358	* hosts (except darwin) for 64-bit guests.
4359	*
4360	* @param pVCpu Pointer to the VMCPU.
4361	* @param idxField The VMCS field encoding.
4362	* @param u64Val 16, 32 or 64 bits value.
4363	*/
4364	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4365	{
4366	AssertPtr(pVCpu);
4367	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4368
4369	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4370	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4371
4372	/* Make sure there are no duplicates. */
4373	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4374	{
4375	if (pCache->Write.aField[i] == idxField)
4376	{
4377	pCache->Write.aFieldVal[i] = u64Val;
4378	return VINF_SUCCESS;
4379	}
4380	}
4381
4382	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4383	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4384	pCache->Write.cValidEntries++;
4385	return VINF_SUCCESS;
4386	}
4387
4388	/* Enable later when the assembly code uses these as callbacks. */
4389	#if 0
4390	/*
4391	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4392	*
4393	* @param pVCpu Pointer to the VMCPU.
4394	* @param pCache Pointer to the VMCS cache.
4395	*
4396	* @remarks No-long-jump zone!!!
4397	*/
4398	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4399	{
4400	AssertPtr(pCache);
4401	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4402	{
4403	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4404	AssertRC(rc);
4405	}
4406	pCache->Write.cValidEntries = 0;
4407	}
4408
4409
4410	/**
4411	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4412	*
4413	* @param pVCpu Pointer to the VMCPU.
4414	* @param pCache Pointer to the VMCS cache.
4415	*
4416	* @remarks No-long-jump zone!!!
4417	*/
4418	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4419	{
4420	AssertPtr(pCache);
4421	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4422	{
4423	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4424	AssertRC(rc);
4425	}
4426	}
4427	#endif
4428	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4429
4430
4431	/**
4432	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4433	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4434	* timer.
4435	*
4436	* @returns VBox status code.
4437	* @param pVCpu Pointer to the VMCPU.
4438	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4439	* out-of-sync. Make sure to update the required fields
4440	* before using them.
4441	* @remarks No-long-jump zone!!!
4442	*/
4443	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4444	{
4445	int rc = VERR_INTERNAL_ERROR_5;
4446	bool fOffsettedTsc = false;
4447	PVM pVM = pVCpu->CTX_SUFF(pVM);
4448	if (pVM->hm.s.vmx.fUsePreemptTimer)
4449	{
4450	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4451
4452	/* Make sure the returned values have sane upper and lower boundaries. */
4453	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4454	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4455	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4456	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4457
4458	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4459	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4460	}
4461	else
4462	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4463
4464	if (fOffsettedTsc)
4465	{
4466	uint64_t u64CurTSC = ASMReadTSC();
4467	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4468	{
4469	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4470	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4471
4472	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4473	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4474	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4475	}
4476	else
4477	{
4478	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4479	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4480	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4481	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4482	}
4483	}
4484	else
4485	{
4486	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4487	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4488	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4489	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4490	}
4491	}
4492
4493
4494	/**
4495	* Determines if an exception is a contributory exception. Contributory
4496	* exceptions are ones which can cause double-faults. Page-fault is
4497	* intentionally not included here as it's a conditional contributory exception.
4498	*
4499	* @returns true if the exception is contributory, false otherwise.
4500	* @param uVector The exception vector.
4501	*/
4502	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4503	{
4504	switch (uVector)
4505	{
4506	case X86_XCPT_GP:
4507	case X86_XCPT_SS:
4508	case X86_XCPT_NP:
4509	case X86_XCPT_TS:
4510	case X86_XCPT_DE:
4511	return true;
4512	default:
4513	break;
4514	}
4515	return false;
4516	}
4517
4518
4519	/**
4520	* Sets an event as a pending event to be injected into the guest.
4521	*
4522	* @param pVCpu Pointer to the VMCPU.
4523	* @param u32IntrInfo The VM-entry interruption-information field.
4524	* @param cbInstr The VM-entry instruction length in bytes (for software
4525	* interrupts, exceptions and privileged software
4526	* exceptions).
4527	* @param u32ErrCode The VM-entry exception error code.
4528	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4529	* page-fault.
4530	*/
4531	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4532	RTGCUINTPTR GCPtrFaultAddress)
4533	{
4534	Assert(!pVCpu->hm.s.Event.fPending);
4535	pVCpu->hm.s.Event.fPending = true;
4536	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4537	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4538	pVCpu->hm.s.Event.cbInstr = cbInstr;
4539	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4540	}
4541
4542
4543	/**
4544	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4545	*
4546	* @param pVCpu Pointer to the VMCPU.
4547	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4548	* out-of-sync. Make sure to update the required fields
4549	* before using them.
4550	*/
4551	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4552	{
4553	/* Inject the double-fault. */
4554	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4555	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4556	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4557	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4558	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4559	}
4560
4561
4562	/**
4563	* Handle a condition that occurred while delivering an event through the guest
4564	* IDT.
4565	*
4566	* @returns VBox status code (informational error codes included).
4567	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4568	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4569	* continue execution of the guest which will delivery the #DF.
4570	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4571	*
4572	* @param pVCpu Pointer to the VMCPU.
4573	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4574	* out-of-sync. Make sure to update the required fields
4575	* before using them.
4576	* @param pVmxTransient Pointer to the VMX transient structure.
4577	*
4578	* @remarks No-long-jump zone!!!
4579	*/
4580	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4581	{
4582	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4583	AssertRC(rc);
4584	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4585	{
4586	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4587	AssertRCReturn(rc, rc);
4588
4589	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4590	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4591	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4592
4593	typedef enum
4594	{
4595	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4596	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4597	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4598	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4599	} VMXREFLECTXCPT;
4600
4601	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4602	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4603	RTGCUINTPTR GCPtrFaultAddress = 0;
4604	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4605	{
4606	enmReflect = VMXREFLECTXCPT_XCPT;
4607	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4608	&& uExitVector == X86_XCPT_PF)
4609	{
4610	GCPtrFaultAddress = pMixedCtx->cr2;
4611	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4612	}
4613	else if ( uExitVector == X86_XCPT_PF
4614	&& uIdtVector == X86_XCPT_PF)
4615	{
4616	pVmxTransient->fVectoringPF = true;
4617	GCPtrFaultAddress = pMixedCtx->cr2;
4618	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4619	}
4620	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4621	&& hmR0VmxIsContributoryXcpt(uExitVector)
4622	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4623	\|\| uIdtVector == X86_XCPT_PF))
4624	{
4625	enmReflect = VMXREFLECTXCPT_DF;
4626	}
4627	else if (uIdtVector == X86_XCPT_DF)
4628	enmReflect = VMXREFLECTXCPT_TF;
4629	}
4630	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4631	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4632	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4633	{
4634	/*
4635	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4636	* (whatever they are) as they reoccur when restarting the instruction.
4637	*/
4638	enmReflect = VMXREFLECTXCPT_XCPT;
4639	}
4640
4641	switch (enmReflect)
4642	{
4643	case VMXREFLECTXCPT_XCPT:
4644	{
4645	uint32_t u32ErrCode = 0;
4646	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4647	{
4648	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4649	AssertRCReturn(rc, rc);
4650	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4651	}
4652	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4653	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4654	rc = VINF_SUCCESS;
4655	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4656	break;
4657	}
4658
4659	case VMXREFLECTXCPT_DF:
4660	{
4661	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4662	rc = VINF_VMX_DOUBLE_FAULT;
4663	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4664	uExitVector));
4665	break;
4666	}
4667
4668	case VMXREFLECTXCPT_TF:
4669	{
4670	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4671	rc = VINF_EM_RESET;
4672	break;
4673	}
4674
4675	default:
4676	Assert(rc == VINF_SUCCESS);
4677	break;
4678	}
4679	}
4680	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4681	return rc;
4682	}
4683
4684
4685	/**
4686	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4687	*
4688	* @returns VBox status code.
4689	* @param pVCpu Pointer to the VMCPU.
4690	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4691	* out-of-sync. Make sure to update the required fields
4692	* before using them.
4693	*
4694	* @remarks No-long-jump zone!!!
4695	*/
4696	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4697	{
4698	int rc = VINF_SUCCESS;
4699	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4700	{
4701	uint32_t uVal = 0;
4702	uint32_t uShadow = 0;
4703	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4704	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4705	AssertRCReturn(rc, rc);
4706	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4707	CPUMSetGuestCR0(pVCpu, uVal);
4708	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4709	}
4710	return rc;
4711	}
4712
4713
4714	/**
4715	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4716	*
4717	* @returns VBox status code.
4718	* @param pVCpu Pointer to the VMCPU.
4719	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4720	* out-of-sync. Make sure to update the required fields
4721	* before using them.
4722	*
4723	* @remarks No-long-jump zone!!!
4724	*/
4725	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4726	{
4727	int rc = VINF_SUCCESS;
4728	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4729	{
4730	uint32_t uVal = 0;
4731	uint32_t uShadow = 0;
4732	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4733	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4734	AssertRCReturn(rc, rc);
4735	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4736	CPUMSetGuestCR4(pVCpu, uVal);
4737	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4738	}
4739	return rc;
4740	}
4741
4742
4743	/**
4744	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4745	*
4746	* @returns VBox status code.
4747	* @param pVCpu Pointer to the VMCPU.
4748	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4749	* out-of-sync. Make sure to update the required fields
4750	* before using them.
4751	*
4752	* @remarks No-long-jump zone!!!
4753	*/
4754	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4755	{
4756	int rc = VINF_SUCCESS;
4757	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4758	{
4759	RTGCUINTREG uVal = 0;
4760	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4761	AssertRCReturn(rc, rc);
4762	pMixedCtx->rip = uVal;
4763	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4764	}
4765	return rc;
4766	}
4767
4768
4769	/**
4770	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4771	*
4772	* @returns VBox status code.
4773	* @param pVCpu Pointer to the VMCPU.
4774	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4775	* out-of-sync. Make sure to update the required fields
4776	* before using them.
4777	*
4778	* @remarks No-long-jump zone!!!
4779	*/
4780	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4781	{
4782	int rc = VINF_SUCCESS;
4783	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4784	{
4785	RTGCUINTREG uVal = 0;
4786	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4787	AssertRCReturn(rc, rc);
4788	pMixedCtx->rsp = uVal;
4789	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4790	}
4791	return rc;
4792	}
4793
4794
4795	/**
4796	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4797	*
4798	* @returns VBox status code.
4799	* @param pVCpu Pointer to the VMCPU.
4800	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4801	* out-of-sync. Make sure to update the required fields
4802	* before using them.
4803	*
4804	* @remarks No-long-jump zone!!!
4805	*/
4806	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4807	{
4808	int rc = VINF_SUCCESS;
4809	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4810	{
4811	uint32_t uVal = 0;
4812	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4813	AssertRCReturn(rc, rc);
4814	pMixedCtx->eflags.u32 = uVal;
4815
4816	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4817	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4818	{
4819	PVM pVM = pVCpu->CTX_SUFF(pVM);
4820	Assert(pVM->hm.s.vmx.pRealModeTSS);
4821	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4822	pMixedCtx->eflags.Bits.u1VM = 0;
4823	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4824	}
4825
4826	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4827	}
4828	return rc;
4829	}
4830
4831
4832	/**
4833	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4834	* guest-CPU context.
4835	*/
4836	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4837	{
4838	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4839	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4840	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4841	return rc;
4842	}
4843
4844
4845	/**
4846	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4847	* from the guest-state area in the VMCS.
4848	*
4849	* @param pVCpu Pointer to the VMCPU.
4850	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4851	* out-of-sync. Make sure to update the required fields
4852	* before using them.
4853	*
4854	* @remarks No-long-jump zone!!!
4855	*/
4856	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4857	{
4858	uint32_t uIntrState = 0;
4859	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4860	AssertRC(rc);
4861
4862	if (!uIntrState)
4863	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4864	else
4865	{
4866	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4867	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4868	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4869	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4870	AssertRC(rc);
4871	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4872	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4873	}
4874	}
4875
4876
4877	/**
4878	* Saves the guest's activity state.
4879	*
4880	* @returns VBox status code.
4881	* @param pVCpu Pointer to the VMCPU.
4882	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4883	* out-of-sync. Make sure to update the required fields
4884	* before using them.
4885	*
4886	* @remarks No-long-jump zone!!!
4887	*/
4888	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4889	{
4890	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4891	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4892	return VINF_SUCCESS;
4893	}
4894
4895
4896	/**
4897	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4898	* the current VMCS into the guest-CPU context.
4899	*
4900	* @returns VBox status code.
4901	* @param pVCpu Pointer to the VMCPU.
4902	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4903	* out-of-sync. Make sure to update the required fields
4904	* before using them.
4905	*
4906	* @remarks No-long-jump zone!!!
4907	*/
4908	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4909	{
4910	int rc = VINF_SUCCESS;
4911	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4912	{
4913	uint32_t u32Val = 0;
4914	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4915	pMixedCtx->SysEnter.cs = u32Val;
4916	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4917	}
4918
4919	RTGCUINTREG uGCVal = 0;
4920	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4921	{
4922	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4923	pMixedCtx->SysEnter.eip = uGCVal;
4924	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4925	}
4926	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4927	{
4928	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4929	pMixedCtx->SysEnter.esp = uGCVal;
4930	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4931	}
4932	return rc;
4933	}
4934
4935
4936	/**
4937	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4938	* context.
4939	*
4940	* @returns VBox status code.
4941	* @param pVCpu Pointer to the VMCPU.
4942	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4943	* out-of-sync. Make sure to update the required fields
4944	* before using them.
4945	*
4946	* @remarks No-long-jump zone!!!
4947	*/
4948	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4949	{
4950	int rc = VINF_SUCCESS;
4951	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4952	{
4953	RTGCUINTREG uVal = 0;
4954	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4955	pMixedCtx->fs.u64Base = uVal;
4956	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4957	}
4958	return rc;
4959	}
4960
4961
4962	/**
4963	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4964	* context.
4965	*
4966	* @returns VBox status code.
4967	* @param pVCpu Pointer to the VMCPU.
4968	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4969	* out-of-sync. Make sure to update the required fields
4970	* before using them.
4971	*
4972	* @remarks No-long-jump zone!!!
4973	*/
4974	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4975	{
4976	int rc = VINF_SUCCESS;
4977	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
4978	{
4979	RTGCUINTREG uVal = 0;
4980	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4981	pMixedCtx->gs.u64Base = uVal;
4982	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
4983	}
4984	return rc;
4985	}
4986
4987
4988	/**
4989	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4990	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4991	*
4992	* @returns VBox status code.
4993	* @param pVCpu Pointer to the VMCPU.
4994	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4995	* out-of-sync. Make sure to update the required fields
4996	* before using them.
4997	*
4998	* @remarks No-long-jump zone!!!
4999	*/
5000	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5001	{
5002	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5003	return VINF_SUCCESS;
5004
5005	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5006	{
5007	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5008	pMsr += i;
5009	switch (pMsr->u32IndexMSR)
5010	{
5011	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5012	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5013	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5014	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5015	#if 0
5016	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5017	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5018	#endif
5019	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5020	default:
5021	{
5022	AssertFailed();
5023	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5024	}
5025	}
5026	}
5027	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5028	return VINF_SUCCESS;
5029	}
5030
5031
5032	/**
5033	* Saves the guest control registers from the current VMCS into the guest-CPU
5034	* context.
5035	*
5036	* @returns VBox status code.
5037	* @param pVCpu Pointer to the VMCPU.
5038	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5039	* out-of-sync. Make sure to update the required fields
5040	* before using them.
5041	*
5042	* @remarks No-long-jump zone!!!
5043	*/
5044	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5045	{
5046	/* Guest CR0. Guest FPU. */
5047	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5048
5049	/* Guest CR4. */
5050	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5051	AssertRCReturn(rc, rc);
5052
5053	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5054	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5055	{
5056	PVM pVM = pVCpu->CTX_SUFF(pVM);
5057	if ( pVM->hm.s.fNestedPaging
5058	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5059	{
5060	RTGCUINTREG uVal = 0;
5061	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5062	if (pMixedCtx->cr3 != uVal)
5063	{
5064	CPUMSetGuestCR3(pVCpu, uVal);
5065	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5066	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5067	}
5068
5069	/* We require EFER to check PAE mode. */
5070	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5071
5072	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5073	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5074	{
5075	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5076	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5077	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5078	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5079	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5080	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5081	}
5082	AssertRCReturn(rc, rc);
5083	}
5084
5085	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5086	}
5087	return rc;
5088	}
5089
5090
5091	/**
5092	* Reads a guest segment register from the current VMCS into the guest-CPU
5093	* context.
5094	*
5095	* @returns VBox status code.
5096	* @param pVCpu Pointer to the VMCPU.
5097	* @param idxSel Index of the selector in the VMCS.
5098	* @param idxLimit Index of the segment limit in the VMCS.
5099	* @param idxBase Index of the segment base in the VMCS.
5100	* @param idxAccess Index of the access rights of the segment in the VMCS.
5101	* @param pSelReg Pointer to the segment selector.
5102	*
5103	* @remarks No-long-jump zone!!!
5104	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5105	* as that takes care of whether to read from the VMCS cache or not.
5106	*/
5107	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5108	PCPUMSELREG pSelReg)
5109	{
5110	uint32_t u32Val = 0;
5111	int rc = VMXReadVmcs32(idxSel, &u32Val);
5112	pSelReg->Sel = (uint16_t)u32Val;
5113	pSelReg->ValidSel = (uint16_t)u32Val;
5114	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5115
5116	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5117	pSelReg->u32Limit = u32Val;
5118
5119	RTGCUINTREG uGCVal = 0;
5120	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5121	pSelReg->u64Base = uGCVal;
5122
5123	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5124	pSelReg->Attr.u = u32Val;
5125	AssertRCReturn(rc, rc);
5126
5127	/*
5128	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5129	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5130	*/
5131	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5132	{
5133	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5134	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5135	}
5136	return rc;
5137	}
5138
5139
5140	/**
5141	* Saves the guest segment registers from the current VMCS into the guest-CPU
5142	* context.
5143	*
5144	* @returns VBox status code.
5145	* @param pVCpu Pointer to the VMCPU.
5146	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5147	* out-of-sync. Make sure to update the required fields
5148	* before using them.
5149	*
5150	* @remarks No-long-jump zone!!!
5151	*/
5152	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5153	{
5154	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5155	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5156	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5157	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5158	#else
5159	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5160	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5161	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5162	#endif
5163
5164	int rc = VINF_SUCCESS;
5165
5166	/* Guest segment registers. */
5167	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5168	{
5169	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5170	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5171	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5172	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5173	rc \|= VMXLOCAL_READ_SEG(ES, es);
5174	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5175	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5176	AssertRCReturn(rc, rc);
5177
5178	/* Restore segment attributes for real-on-v86 mode hack. */
5179	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5180	{
5181	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5182	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5183	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5184	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5185	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5186	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5187	}
5188	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5189	}
5190
5191	/* Guest LDTR. */
5192	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5193	{
5194	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5195	AssertRCReturn(rc, rc);
5196	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5197	}
5198
5199	/* Guest GDTR. */
5200	RTGCUINTREG uGCVal = 0;
5201	uint32_t u32Val = 0;
5202	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5203	{
5204	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5205	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5206	pMixedCtx->gdtr.pGdt = uGCVal;
5207	pMixedCtx->gdtr.cbGdt = u32Val;
5208	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5209	}
5210
5211	/* Guest IDTR. */
5212	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5213	{
5214	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5215	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5216	pMixedCtx->idtr.pIdt = uGCVal;
5217	pMixedCtx->idtr.cbIdt = u32Val;
5218	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5219	}
5220
5221	/* Guest TR. */
5222	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5223	{
5224	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5225
5226	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5227	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5228	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5229	AssertRCReturn(rc, rc);
5230	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5231	}
5232	return rc;
5233	}
5234
5235
5236	/**
5237	* Saves the guest debug registers from the current VMCS into the guest-CPU
5238	* context.
5239	*
5240	* @returns VBox status code.
5241	* @param pVCpu Pointer to the VMCPU.
5242	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5243	* out-of-sync. Make sure to update the required fields
5244	* before using them.
5245	*
5246	* @remarks No-long-jump zone!!!
5247	*/
5248	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5249	{
5250	int rc = VINF_SUCCESS;
5251	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5252	{
5253	RTGCUINTREG uVal;
5254	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5255	pMixedCtx->dr[7] = uVal;
5256
5257	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5258	}
5259	return rc;
5260	}
5261
5262
5263	/**
5264	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5265	*
5266	* @returns VBox status code.
5267	* @param pVCpu Pointer to the VMCPU.
5268	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5269	* out-of-sync. Make sure to update the required fields
5270	* before using them.
5271	*
5272	* @remarks No-long-jump zone!!!
5273	*/
5274	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5275	{
5276	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5277	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5278	return VINF_SUCCESS;
5279	}
5280
5281
5282	/**
5283	* Saves the entire guest state from the currently active VMCS into the
5284	* guest-CPU context. This essentially VMREADs all guest-data.
5285	*
5286	* @returns VBox status code.
5287	* @param pVCpu Pointer to the VMCPU.
5288	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5289	* out-of-sync. Make sure to update the required fields
5290	* before using them.
5291	*/
5292	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5293	{
5294	Assert(pVCpu);
5295	Assert(pMixedCtx);
5296
5297	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5298	return VINF_SUCCESS;
5299
5300	VMMRZCallRing3Disable(pVCpu);
5301	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5302	LogFunc(("\n"));
5303
5304	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5305	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5306
5307	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5308	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5309
5310	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5311	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5312
5313	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5314	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5315
5316	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5317	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5318
5319	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5320	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5321
5322	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5324
5325	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5327
5328	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5330
5331	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5332	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5333
5334	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5335	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5336
5337	VMMRZCallRing3Enable(pVCpu);
5338	return rc;
5339	}
5340
5341
5342	/**
5343	* Check per-VM and per-VCPU force flag actions that require us to go back to
5344	* ring-3 for one reason or another.
5345	*
5346	* @returns VBox status code (information status code included).
5347	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5348	* ring-3.
5349	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5350	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5351	* interrupts)
5352	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5353	* all EMTs to be in ring-3.
5354	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5355	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5356	* to the EM loop.
5357	*
5358	* @param pVM Pointer to the VM.
5359	* @param pVCpu Pointer to the VMCPU.
5360	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5361	* out-of-sync. Make sure to update the required fields
5362	* before using them.
5363	*/
5364	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5365	{
5366	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5367
5368	int rc = VERR_INTERNAL_ERROR_5;
5369	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5370	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5371	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5372	{
5373	/* We need the control registers now, make sure the guest-CPU context is updated. */
5374	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5375	AssertRCReturn(rc, rc);
5376
5377	/* Pending HM CR3 sync. */
5378	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5379	{
5380	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5381	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5382	}
5383	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5384	{
5385	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5386	AssertRC(rc);
5387	}
5388
5389	/* Pending PGM C3 sync. */
5390	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5391	{
5392	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5393	if (rc != VINF_SUCCESS)
5394	{
5395	AssertRC(rc);
5396	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5397	return rc;
5398	}
5399	}
5400
5401	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5402	/* -XXX- what was that about single stepping? */
5403	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5404	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5405	{
5406	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5407	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5408	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5409	return rc;
5410	}
5411
5412	/* Pending VM request packets, such as hardware interrupts. */
5413	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5414	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5415	{
5416	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5417	return VINF_EM_PENDING_REQUEST;
5418	}
5419
5420	/* Pending PGM pool flushes. */
5421	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5422	{
5423	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5424	return VINF_PGM_POOL_FLUSH_PENDING;
5425	}
5426
5427	/* Pending DMA requests. */
5428	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5429	{
5430	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5431	return VINF_EM_RAW_TO_R3;
5432	}
5433	}
5434
5435	/* Paranoia. */
5436	Assert(rc != VERR_EM_INTERPRETER);
5437	return VINF_SUCCESS;
5438	}
5439
5440
5441	/**
5442	* Converts any TRPM trap into a pending VMX event. This is typically used when
5443	* entering from ring-3 (not longjmp returns).
5444	*
5445	* @param pVCpu Pointer to the VMCPU.
5446	* @param pCtx Pointer to the guest-CPU context.
5447	*/
5448	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5449	{
5450	if (!TRPMHasTrap(pVCpu))
5451	{
5452	Assert(!pVCpu->hm.s.Event.fPending);
5453	return;
5454	}
5455
5456	uint8_t uVector;
5457	TRPMEVENT enmTrpmEvent;
5458	RTGCUINT uErrCode;
5459	RTGCUINTPTR GCPtrFaultAddress;
5460	uint8_t cbInstr;
5461
5462	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5463	AssertRC(rc);
5464
5465	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5466	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5467	if (enmTrpmEvent == TRPM_TRAP)
5468	{
5469	switch (uVector)
5470	{
5471	case X86_XCPT_BP:
5472	case X86_XCPT_OF:
5473	{
5474	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5475	break;
5476	}
5477
5478	case X86_XCPT_PF:
5479	case X86_XCPT_DF:
5480	case X86_XCPT_TS:
5481	case X86_XCPT_NP:
5482	case X86_XCPT_SS:
5483	case X86_XCPT_GP:
5484	case X86_XCPT_AC:
5485	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5486	/* no break! */
5487	default:
5488	{
5489	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5490	break;
5491	}
5492	}
5493	}
5494	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5495	{
5496	if (uVector != X86_XCPT_NMI)
5497	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5498	else
5499	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5500	}
5501	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5502	{
5503	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5504	}
5505	else
5506	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5507
5508	rc = TRPMResetTrap(pVCpu);
5509	AssertRC(rc);
5510	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5511	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5512	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5513	}
5514
5515
5516	/**
5517	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5518	* VT-x to execute any instruction.
5519	*
5520	* @param pvCpu Pointer to the VMCPU.
5521	*/
5522	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5523	{
5524	if (pVCpu->hm.s.Event.fPending)
5525	{
5526	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5527	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5528	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5529	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5530
5531	/* If a trap was already pending, we did something wrong! */
5532	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5533
5534	TRPMEVENT enmTrapType;
5535	switch (uVectorType)
5536	{
5537	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5538	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5539	enmTrapType = TRPM_HARDWARE_INT;
5540	break;
5541	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5542	enmTrapType = TRPM_SOFTWARE_INT;
5543	break;
5544	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5545	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5546	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5547	enmTrapType = TRPM_TRAP;
5548	break;
5549	default:
5550	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5551	enmTrapType = TRPM_32BIT_HACK;
5552	break;
5553	}
5554
5555	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5556	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5557	AssertRC(rc);
5558
5559	if (fErrorCodeValid)
5560	TRPMSetErrorCode(pVCpu, uErrorCode);
5561	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5562	&& uVector == X86_XCPT_PF)
5563	{
5564	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5565	}
5566	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5567	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5568	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5569	{
5570	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5571	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5572	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5573	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5574	}
5575	pVCpu->hm.s.Event.fPending = false;
5576	}
5577	}
5578
5579
5580	/**
5581	* Does the necessary state syncing before doing a longjmp to ring-3.
5582	*
5583	* @param pVM Pointer to the VM.
5584	* @param pVCpu Pointer to the VMCPU.
5585	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5586	* out-of-sync. Make sure to update the required fields
5587	* before using them.
5588	* @param rcExit The reason for exiting to ring-3. Can be
5589	* VINF_VMM_UNKNOWN_RING3_CALL.
5590	*
5591	* @remarks No-long-jmp zone!!!
5592	*/
5593	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5594	{
5595	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5596	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5597
5598	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5599	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5600	AssertRC(rc);
5601
5602	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5603	if (CPUMIsGuestFPUStateActive(pVCpu))
5604	{
5605	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5606	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5607	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5608	}
5609
5610	/* Restore debug registers if necessary and resync on next R0 reentry. */
5611	if (CPUMIsGuestDebugStateActive(pVCpu))
5612	{
5613	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5614	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5615	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5616	}
5617	else if (CPUMIsHyperDebugStateActive(pVCpu))
5618	{
5619	CPUMR0LoadHostDebugState(pVM, pVCpu);
5620	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5621	}
5622
5623	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5624	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5625	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5626	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5627	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5628	}
5629
5630
5631	/**
5632	* An action requires us to go back to ring-3. This function does the necessary
5633	* steps before we can safely return to ring-3. This is not the same as longjmps
5634	* to ring-3, this is voluntary.
5635	*
5636	* @param pVM Pointer to the VM.
5637	* @param pVCpu Pointer to the VMCPU.
5638	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5639	* out-of-sync. Make sure to update the required fields
5640	* before using them.
5641	* @param rcExit The reason for exiting to ring-3. Can be
5642	* VINF_VMM_UNKNOWN_RING3_CALL.
5643	*/
5644	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5645	{
5646	Assert(pVM);
5647	Assert(pVCpu);
5648	Assert(pMixedCtx);
5649	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5650
5651	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5652	{
5653	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5654	return;
5655	}
5656	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5657	{
5658	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5659	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5660	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5661	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5662	return;
5663	}
5664
5665	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5666	VMMRZCallRing3Disable(pVCpu);
5667	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5668
5669	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5670	hmR0VmxUpdateTRPM(pVCpu);
5671
5672	/* Sync. the guest state. */
5673	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5674	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5675
5676	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5677	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5678	\| CPUM_CHANGED_LDTR
5679	\| CPUM_CHANGED_GDTR
5680	\| CPUM_CHANGED_IDTR
5681	\| CPUM_CHANGED_TR
5682	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5683
5684	/* On our way back from ring-3 the following needs to be done. */
5685	/** @todo This can change with preemption hooks. */
5686	if (rcExit == VINF_EM_RAW_INTERRUPT)
5687	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5688	else
5689	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5690
5691	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5692	VMMRZCallRing3Enable(pVCpu);
5693	}
5694
5695
5696	/**
5697	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5698	* longjump to ring-3 and possibly get preempted.
5699	*
5700	* @param pVCpu Pointer to the VMCPU.
5701	* @param enmOperation The operation causing the ring-3 longjump.
5702	* @param pvUser The user argument (pointer to the possibly
5703	* out-of-date guest-CPU context).
5704	*
5705	* @remarks Must never be called with @a enmOperation ==
5706	* VMMCALLRING3_VM_R0_ASSERTION.
5707	*/
5708	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5709	{
5710	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5711	Assert(pVCpu);
5712	Assert(pvUser);
5713	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5714	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5715
5716	VMMRZCallRing3Disable(pVCpu);
5717	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5718	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5719	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5720	VMMRZCallRing3Enable(pVCpu);
5721	}
5722
5723
5724	/**
5725	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5726	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5727	*
5728	* @returns VBox status code.
5729	* @param pVCpu Pointer to the VMCPU.
5730	*/
5731	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5732	{
5733	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5734	{
5735	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5736	{
5737	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5738	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5739	AssertRC(rc);
5740	}
5741	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5742	}
5743
5744
5745	/**
5746	* Injects any pending events into the guest if the guest is in a state to
5747	* receive them.
5748	*
5749	* @returns VBox status code (informational status codes included).
5750	* @param pVCpu Pointer to the VMCPU.
5751	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5752	* out-of-sync. Make sure to update the required fields
5753	* before using them.
5754	*/
5755	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5756	{
5757	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5758	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5759	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5760	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5761
5762	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5763	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5764	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5765	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5766
5767	int rc = VINF_SUCCESS;
5768	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5769	{
5770	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5771	bool fInject = true;
5772	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5773	{
5774	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5775	AssertRCReturn(rc, rc);
5776	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5777	if ( fBlockInt
5778	\|\| fBlockSti
5779	\|\| fBlockMovSS)
5780	{
5781	fInject = false;
5782	}
5783	}
5784	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5785	&& ( fBlockMovSS
5786	\|\| fBlockSti))
5787	{
5788	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5789	fInject = false;
5790	}
5791
5792	if (fInject)
5793	{
5794	Log(("Injecting pending event\n"));
5795	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5796	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5797	AssertRCReturn(rc, rc);
5798	pVCpu->hm.s.Event.fPending = false;
5799	}
5800	else
5801	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5802	} /** @todo SMI. SMIs take priority over NMIs. */
5803	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5804	{
5805	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5806	if ( !fBlockMovSS
5807	&& !fBlockSti)
5808	{
5809	Log(("Injecting NMI\n"));
5810	RTGCUINTPTR uIntrInfo;
5811	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5812	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5813	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5814	0 /* GCPtrFaultAddress */, &uIntrState);
5815	AssertRCReturn(rc, rc);
5816	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5817	}
5818	else
5819	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5820	}
5821	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5822	{
5823	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5824	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5825	AssertRCReturn(rc, rc);
5826	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5827	if ( !fBlockInt
5828	&& !fBlockSti
5829	&& !fBlockMovSS)
5830	{
5831	uint8_t u8Interrupt;
5832	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5833	if (RT_SUCCESS(rc))
5834	{
5835	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5836	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5837	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5838	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5839	0 /* GCPtrFaultAddress */, &uIntrState);
5840	}
5841	else
5842	{
5843	/** @todo Does this actually happen? If not turn it into an assertion. */
5844	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5845	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5846	rc = VINF_SUCCESS;
5847	}
5848	}
5849	else
5850	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5851	}
5852
5853	/*
5854	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5855	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5856	*/
5857	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5858	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5859	int rc2 = VINF_SUCCESS;
5860	if ( fBlockSti
5861	\|\| fBlockMovSS)
5862	{
5863	if (!DBGFIsStepping(pVCpu))
5864	{
5865	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5866	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5867	{
5868	/*
5869	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5870	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5871	*/
5872	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5873	}
5874	}
5875	else
5876	{
5877	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5878	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5879	uIntrState = 0;
5880	}
5881	}
5882
5883	/*
5884	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5885	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5886	*/
5887	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5888	AssertRC(rc2);
5889
5890	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5891	return rc;
5892	}
5893
5894
5895	/**
5896	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5897	*
5898	* @param pVCpu Pointer to the VMCPU.
5899	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5900	* out-of-sync. Make sure to update the required fields
5901	* before using them.
5902	*/
5903	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5904	{
5905	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5906	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5907	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5908	}
5909
5910
5911	/**
5912	* Injects a double-fault (#DF) exception into the VM.
5913	*
5914	* @returns VBox status code (informational status code included).
5915	* @param pVCpu Pointer to the VMCPU.
5916	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5917	* out-of-sync. Make sure to update the required fields
5918	* before using them.
5919	*/
5920	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5921	{
5922	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5923	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5924	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5925	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5926	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5927	puIntrState);
5928	}
5929
5930
5931	/**
5932	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5933	*
5934	* @param pVCpu Pointer to the VMCPU.
5935	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5936	* out-of-sync. Make sure to update the required fields
5937	* before using them.
5938	*/
5939	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5940	{
5941	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5942	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5943	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5944	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5945	}
5946
5947
5948	/**
5949	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
5950	*
5951	* @param pVCpu Pointer to the VMCPU.
5952	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5953	* out-of-sync. Make sure to update the required fields
5954	* before using them.
5955	* @param cbInstr The value of RIP that is to be pushed on the guest
5956	* stack.
5957	*/
5958	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5959	{
5960	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5961	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5962	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5963	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
5964	}
5965
5966
5967	/**
5968	* Injects a general-protection (#GP) fault into the VM.
5969	*
5970	* @returns VBox status code (informational status code included).
5971	* @param pVCpu Pointer to the VMCPU.
5972	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5973	* out-of-sync. Make sure to update the required fields
5974	* before using them.
5975	* @param u32ErrorCode The error code associated with the #GP.
5976	*/
5977	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
5978	uint32_t *puIntrState)
5979	{
5980	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5981	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5982	if (fErrorCodeValid)
5983	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5984	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5985	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
5986	puIntrState);
5987	}
5988
5989
5990	/**
5991	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
5992	*
5993	* @param pVCpu Pointer to the VMCPU.
5994	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5995	* out-of-sync. Make sure to update the required fields
5996	* before using them.
5997	* @param uVector The software interrupt vector number.
5998	* @param cbInstr The value of RIP that is to be pushed on the guest
5999	* stack.
6000	*/
6001	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6002	{
6003	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6004	if ( uVector == X86_XCPT_BP
6005	\|\| uVector == X86_XCPT_OF)
6006	{
6007	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6008	}
6009	else
6010	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6011	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6012	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6013	}
6014
6015
6016	/**
6017	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6018	* stack.
6019	*
6020	* @returns VBox status code (information status code included).
6021	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6022	* @param pVM Pointer to the VM.
6023	* @param pMixedCtx Pointer to the guest-CPU context.
6024	* @param uValue The value to push to the guest stack.
6025	*/
6026	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6027	{
6028	/*
6029	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6030	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6031	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6032	*/
6033	if (pMixedCtx->sp == 1)
6034	return VINF_EM_RESET;
6035	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6036	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6037	AssertRCReturn(rc, rc);
6038	return rc;
6039	}
6040
6041
6042	/**
6043	* Injects an event into the guest upon VM-entry by updating the relevant fields
6044	* in the VM-entry area in the VMCS.
6045	*
6046	* @returns VBox status code (informational error codes included).
6047	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6048	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6049	*
6050	* @param pVCpu Pointer to the VMCPU.
6051	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6052	* be out-of-sync. Make sure to update the required
6053	* fields before using them.
6054	* @param u64IntrInfo The VM-entry interruption-information field.
6055	* @param cbInstr The VM-entry instruction length in bytes (for
6056	* software interrupts, exceptions and privileged
6057	* software exceptions).
6058	* @param u32ErrCode The VM-entry exception error code.
6059	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6060	* @param puIntrState Pointer to the current guest interruptibility-state.
6061	* This interruptibility-state will be updated if
6062	* necessary. This cannot not be NULL.
6063	*
6064	* @remarks No-long-jump zone!!!
6065	* @remarks Requires CR0!
6066	*/
6067	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6068	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6069	{
6070	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6071	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6072	Assert(puIntrState);
6073	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6074
6075	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6076	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6077
6078	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6079	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6080	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6081
6082	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6083
6084	/* We require CR0 to check if the guest is in real-mode. */
6085	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6086	AssertRCReturn(rc, rc);
6087
6088	/*
6089	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6090	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6091	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6092	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6093	*/
6094	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6095	{
6096	PVM pVM = pVCpu->CTX_SUFF(pVM);
6097	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6098	{
6099	Assert(PDMVmmDevHeapIsEnabled(pVM));
6100	Assert(pVM->hm.s.vmx.pRealModeTSS);
6101
6102	/* Save the required guest state bits from the VMCS. */
6103	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6104	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6105	AssertRCReturn(rc, rc);
6106
6107	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6108	const size_t cbIdtEntry = 4;
6109	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6110	{
6111	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6112	if (uVector == X86_XCPT_DF)
6113	return VINF_EM_RESET;
6114	else if (uVector == X86_XCPT_GP)
6115	{
6116	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6117	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6118	}
6119
6120	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6121	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6122	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6123	}
6124
6125	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6126	uint16_t uGuestIp = pMixedCtx->ip;
6127	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6128	{
6129	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6130	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6131	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6132	}
6133	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6134	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6135
6136	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6137	uint16_t offIdtEntry = 0;
6138	RTSEL selIdtEntry = 0;
6139	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6140	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6141	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6142	AssertRCReturn(rc, rc);
6143
6144	/* Construct the stack frame for the interrupt/exception handler. */
6145	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6146	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6147	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6148	AssertRCReturn(rc, rc);
6149
6150	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6151	if (rc == VINF_SUCCESS)
6152	{
6153	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6154	pMixedCtx->rip = offIdtEntry;
6155	pMixedCtx->cs.Sel = selIdtEntry;
6156	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6157	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6158	&& uVector == X86_XCPT_PF)
6159	{
6160	pMixedCtx->cr2 = GCPtrFaultAddress;
6161	}
6162	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6163	\| HM_CHANGED_GUEST_RIP
6164	\| HM_CHANGED_GUEST_RFLAGS
6165	\| HM_CHANGED_GUEST_RSP;
6166
6167	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6168	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6169	{
6170	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6171	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6172	Log(("Clearing inhibition due to STI.\n"));
6173	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6174	}
6175	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6176	}
6177	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6178	return rc;
6179	}
6180	else
6181	{
6182	/*
6183	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6184	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6185	*/
6186	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6187	}
6188	}
6189
6190	/* Validate. */
6191	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6192	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6193	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6194
6195	/* Inject. */
6196	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6197	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6198	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6199	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6200
6201	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6202	&& uVector == X86_XCPT_PF)
6203	{
6204	pMixedCtx->cr2 = GCPtrFaultAddress;
6205	}
6206	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6207
6208	AssertRCReturn(rc, rc);
6209	return rc;
6210	}
6211
6212
6213	/**
6214	* Enters the VT-x session.
6215	*
6216	* @returns VBox status code.
6217	* @param pVM Pointer to the VM.
6218	* @param pVCpu Pointer to the VMCPU.
6219	* @param pCpu Pointer to the CPU info struct.
6220	*/
6221	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6222	{
6223	AssertPtr(pVM);
6224	AssertPtr(pVCpu);
6225	Assert(pVM->hm.s.vmx.fSupported);
6226	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6227	NOREF(pCpu);
6228
6229	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6230
6231	/* Make sure we're in VMX root mode. */
6232	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6233	if (!(u32HostCR4 & X86_CR4_VMXE))
6234	{
6235	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6236	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6237	}
6238
6239	/* Load the active VMCS as the current one. */
6240	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6241	if (RT_FAILURE(rc))
6242	return rc;
6243
6244	/** @todo this will change with preemption hooks where can can VMRESUME as long
6245	* as we're no preempted. */
6246	pVCpu->hm.s.fResumeVM = false;
6247	return VINF_SUCCESS;
6248	}
6249
6250
6251	/**
6252	* Leaves the VT-x session.
6253	*
6254	* @returns VBox status code.
6255	* @param pVM Pointer to the VM.
6256	* @param pVCpu Pointer to the VMCPU.
6257	* @param pCtx Pointer to the guest-CPU context.
6258	*/
6259	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6260	{
6261	AssertPtr(pVCpu);
6262	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6263	NOREF(pVM);
6264	NOREF(pCtx);
6265
6266	/** @todo this will change with preemption hooks where we only VMCLEAR when
6267	* we are actually going to be preempted, not all the time like we
6268	* currently do. */
6269	/*
6270	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6271	* and mark the VMCS launch-state as "clear".
6272	*/
6273	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6274	return rc;
6275	}
6276
6277
6278	/**
6279	* Saves the host state in the VMCS host-state.
6280	* Sets up the VM-exit MSR-load area.
6281	*
6282	* The CPU state will be loaded from these fields on every successful VM-exit.
6283	*
6284	* @returns VBox status code.
6285	* @param pVM Pointer to the VM.
6286	* @param pVCpu Pointer to the VMCPU.
6287	*
6288	* @remarks No-long-jump zone!!!
6289	*/
6290	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6291	{
6292	AssertPtr(pVM);
6293	AssertPtr(pVCpu);
6294	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6295
6296	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6297
6298	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6299	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6300	return VINF_SUCCESS;
6301
6302	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6303	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6304
6305	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6306	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6307
6308	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6309	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6310
6311	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6312	return rc;
6313	}
6314
6315
6316	/**
6317	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6318	* loaded from these fields on every successful VM-entry.
6319	*
6320	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6321	* Sets up the VM-entry controls.
6322	* Sets up the appropriate VMX non-root function to execute guest code based on
6323	* the guest CPU mode.
6324	*
6325	* @returns VBox status code.
6326	* @param pVM Pointer to the VM.
6327	* @param pVCpu Pointer to the VMCPU.
6328	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6329	* out-of-sync. Make sure to update the required fields
6330	* before using them.
6331	*
6332	* @remarks No-long-jump zone!!!
6333	*/
6334	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6335	{
6336	AssertPtr(pVM);
6337	AssertPtr(pVCpu);
6338	AssertPtr(pMixedCtx);
6339	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6340
6341	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6342
6343	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6344
6345	/* Determine real-on-v86 mode. */
6346	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6347	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6348	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6349	{
6350	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6351	}
6352
6353	/** @todo if the order of loading is important, inform it via comments here */
6354	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6355	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6356
6357	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6358	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6359
6360	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6361	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6362
6363	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6364	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6365
6366	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6367	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6368
6369	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6370	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6371
6372	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6373	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6374
6375	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6377
6378	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6380
6381	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6383
6384	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6385	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6386	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6387
6388	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6389	return rc;
6390	}
6391
6392
6393	/**
6394	* Does the preparations before executing guest code in VT-x.
6395	*
6396	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6397	* recompiler. We must be cautious what we do here regarding committing
6398	* guest-state information into the the VMCS assuming we assuredly execute the
6399	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6400	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6401	* that the recompiler can (and should) use them when it resumes guest
6402	* execution. Otherwise such operations must be done when we can no longer
6403	* exit to ring-3.
6404	*
6405	* @returns VBox status code (informational status codes included).
6406	* @retval VINF_SUCCESS if we can proceed with running the guest.
6407	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6408	* into the guest.
6409	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6410	*
6411	* @param pVM Pointer to the VM.
6412	* @param pVCpu Pointer to the VMCPU.
6413	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6414	* out-of-sync. Make sure to update the required fields
6415	* before using them.
6416	* @param pVmxTransient Pointer to the VMX transient structure.
6417	*
6418	* @remarks Called with preemption disabled.
6419	*/
6420	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6421	{
6422	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6423
6424	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6425	PGMRZDynMapFlushAutoSet(pVCpu);
6426	#endif
6427
6428	/* Check force flag actions that might require us to go back to ring-3. */
6429	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6430	if (rc != VINF_SUCCESS)
6431	return rc;
6432
6433	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6434	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6435	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6436	{
6437	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6438	RTGCPHYS GCPhysApicBase;
6439	GCPhysApicBase = pMixedCtx->msrApicBase;
6440	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6441
6442	/* Unalias any existing mapping. */
6443	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6444	AssertRCReturn(rc, rc);
6445
6446	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6447	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6448	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6449	AssertRCReturn(rc, rc);
6450
6451	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6452	}
6453
6454	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6455	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6456	pVmxTransient->uEFlags = ASMIntDisableFlags();
6457	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6458	{
6459	ASMSetFlags(pVmxTransient->uEFlags);
6460	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6461	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6462	return VINF_EM_RAW_INTERRUPT;
6463	}
6464	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6465	#endif
6466
6467	/*
6468	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6469	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6470	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6471	*/
6472	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6473	AssertRCReturn(rc, rc);
6474	return rc;
6475	}
6476
6477
6478	/**
6479	* Prepares to run guest code in VT-x and we've committed to doing so. This
6480	* means there is no backing out to ring-3 or anywhere else at this
6481	* point.
6482	*
6483	* @param pVM Pointer to the VM.
6484	* @param pVCpu Pointer to the VMCPU.
6485	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6486	* out-of-sync. Make sure to update the required fields
6487	* before using them.
6488	* @param pVmxTransient Pointer to the VMX transient structure.
6489	*
6490	* @remarks Called with preemption disabled.
6491	* @remarks No-long-jump zone!!!
6492	*/
6493	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6494	{
6495	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6496	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6497
6498	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6499	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6500	pVmxTransient->uEFlags = ASMIntDisableFlags();
6501	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6502	#endif
6503
6504	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6505	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6506	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6507	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6508	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6509	#endif
6510	int rc = VINF_SUCCESS;
6511	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6512	{
6513	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6514	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6515	}
6516	else if (pVCpu->hm.s.fContextUseFlags)
6517	{
6518	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6519	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6520	}
6521	AssertRC(rc);
6522	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6523
6524	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6525	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6526	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6527
6528	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6529	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6530
6531	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6532	{
6533	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6534	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6535	}
6536
6537	/*
6538	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6539	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6540	*/
6541	if (pVM->hm.s.fTPRPatchingActive)
6542	{
6543	Assert(!CPUMIsGuestInLongMode(pVCpu));
6544
6545	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6546	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6547	AssertRC(rc);
6548
6549	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6550	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6551	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6552	see hmR0VmxLoadGuestApicState(). */
6553	}
6554
6555	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6556	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6557	to start executing. */
6558	}
6559
6560
6561	/**
6562	* Performs some essential restoration of state after running guest code in
6563	* VT-x.
6564	*
6565	* @param pVM Pointer to the VM.
6566	* @param pVCpu Pointer to the VMCPU.
6567	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6568	* out-of-sync. Make sure to update the required fields
6569	* before using them.
6570	* @param pVmxTransient Pointer to the VMX transient structure.
6571	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6572	*
6573	* @remarks Called with interrupts disabled.
6574	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6575	* unconditionally when it is safe to do so.
6576	*/
6577	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6578	{
6579	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6580	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6581
6582	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6583	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6584	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6585	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6586	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6587
6588	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6589	{
6590	/** @todo Find a way to fix hardcoding a guestimate. */
6591	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6592	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6593	}
6594
6595	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6596	Assert(!(ASMGetFlags() & X86_EFL_IF));
6597	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6598
6599	/* Restore the effects of TPR patching if any. */
6600	if (pVM->hm.s.fTPRPatchingActive)
6601	{
6602	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6603	AssertRC(rc);
6604	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6605	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6606	}
6607
6608	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6609	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6610
6611	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6612	uint32_t uExitReason;
6613	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6614	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6615	AssertRC(rc);
6616	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6617	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6618
6619	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6620	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6621
6622	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6623	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6624	{
6625	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6626	return;
6627	}
6628
6629	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6630	{
6631	/* Update the guest interruptibility-state from the VMCS. */
6632	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6633	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6634	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6635	AssertRC(rc);
6636	#endif
6637	/*
6638	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6639	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6640	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6641	*/
6642	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6643	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6644	{
6645	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6646	AssertRC(rc);
6647	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6648	}
6649	}
6650	}
6651
6652
6653	/**
6654	* Runs the guest code using VT-x.
6655	*
6656	* @returns VBox status code.
6657	* @param pVM Pointer to the VM.
6658	* @param pVCpu Pointer to the VMCPU.
6659	* @param pCtx Pointer to the guest-CPU context.
6660	*
6661	* @remarks Called with preemption disabled.
6662	*/
6663	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6664	{
6665	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6666	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6667
6668	VMXTRANSIENT VmxTransient;
6669	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6670	int rc = VERR_INTERNAL_ERROR_5;
6671	uint32_t cLoops = 0;
6672	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6673
6674	for (;; cLoops++)
6675	{
6676	Assert(!HMR0SuspendPending());
6677	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6678	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6679	(unsigned)RTMpCpuId(), cLoops));
6680
6681	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6682	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6683	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6684	if (rc != VINF_SUCCESS)
6685	break;
6686
6687	/*
6688	* No longjmps to ring-3 from this point on!!!
6689	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6690	* This also disables flushing of the R0-logger instance (if any).
6691	*/
6692	VMMRZCallRing3Disable(pVCpu);
6693	VMMRZCallRing3RemoveNotification(pVCpu);
6694	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6695
6696	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6697	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6698
6699	/*
6700	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6701	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6702	*/
6703	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6704	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6705	{
6706	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6707	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6708	return rc;
6709	}
6710
6711	/* Handle the VM-exit. */
6712	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6713	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6714	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6715	#ifdef HM_PROFILE_EXIT_DISPATCH
6716	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6717	#endif
6718	#ifdef HMVMX_USE_FUNCTION_TABLE
6719	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6720	#else
6721	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6722	#endif
6723	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6724	if (rc != VINF_SUCCESS)
6725	break;
6726	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6727	{
6728	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6729	rc = VINF_EM_RAW_INTERRUPT;
6730	break;
6731	}
6732	}
6733
6734	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6735	if (rc == VERR_EM_INTERPRETER)
6736	rc = VINF_EM_RAW_EMULATE_INSTR;
6737	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6738	return rc;
6739	}
6740
6741
6742	#ifndef HMVMX_USE_FUNCTION_TABLE
6743	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6744	{
6745	int rc;
6746	switch (rcReason)
6747	{
6748	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6749	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6750	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6751	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6752	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6753	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6754	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6755	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6756	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6757	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6758	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6759	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6760	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6761	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6762	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6763	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6764	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6765	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6766	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6767	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6768	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6769	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6770	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6771	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6772	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6773	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6774	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6775	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6776	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6777	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6778	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6779	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6780	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6781
6782	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6783	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6784	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6785	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6786	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6787	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6788	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6789	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6790	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6791
6792	case VMX_EXIT_VMCALL:
6793	case VMX_EXIT_VMCLEAR:
6794	case VMX_EXIT_VMLAUNCH:
6795	case VMX_EXIT_VMPTRLD:
6796	case VMX_EXIT_VMPTRST:
6797	case VMX_EXIT_VMREAD:
6798	case VMX_EXIT_VMRESUME:
6799	case VMX_EXIT_VMWRITE:
6800	case VMX_EXIT_VMXOFF:
6801	case VMX_EXIT_VMXON:
6802	case VMX_EXIT_INVEPT:
6803	case VMX_EXIT_INVVPID:
6804	case VMX_EXIT_VMFUNC:
6805	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6806	break;
6807	default:
6808	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6809	break;
6810	}
6811	return rc;
6812	}
6813	#endif
6814
6815
6816	/** Profiling macro. */
6817	#ifdef HM_PROFILE_EXIT_DISPATCH
6818	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6819	#else
6820	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6821	#endif
6822
6823
6824	#ifdef DEBUG
6825	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6826	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6827	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6828	# define VMX_ASSERT_PREEMPT_CPUID() \
6829	do \
6830	{ \
6831	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6832	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6833	} while (0)
6834
6835	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6836	do { \
6837	AssertPtr(pVCpu); \
6838	AssertPtr(pMixedCtx); \
6839	AssertPtr(pVmxTransient); \
6840	Assert(pVmxTransient->fVMEntryFailed == false); \
6841	Assert(ASMIntAreEnabled()); \
6842	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6843	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6844	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6845	(unsigned)pVCpu->idCpu)); \
6846	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6847	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6848	VMX_ASSERT_PREEMPT_CPUID(); \
6849	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6850	} while (0)
6851	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6852	do { \
6853	LogFunc(("\n")); \
6854	} while(0)
6855	#else /* Release builds */
6856	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6857	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6858	#endif
6859
6860
6861	/**
6862	* Advances the guest RIP after reading it from the VMCS.
6863	*
6864	* @returns VBox status code.
6865	* @param pVCpu Pointer to the VMCPU.
6866	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6867	* out-of-sync. Make sure to update the required fields
6868	* before using them.
6869	* @param pVmxTransient Pointer to the VMX transient structure.
6870	*
6871	* @remarks No-long-jump zone!!!
6872	*/
6873	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6874	{
6875	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6876	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6877	AssertRCReturn(rc, rc);
6878
6879	pMixedCtx->rip += pVmxTransient->cbInstr;
6880	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6881	return rc;
6882	}
6883
6884
6885	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6886	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6887	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6888	/**
6889	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6890	*/
6891	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6892	{
6893	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6894	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6895	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6896	Assert(ASMIntAreEnabled());
6897	return VINF_SUCCESS;
6898	#else
6899	return VINF_EM_RAW_INTERRUPT;
6900	#endif
6901	}
6902
6903
6904	/**
6905	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6906	*/
6907	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6908	{
6909	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6910	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6911
6912	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6913	AssertRCReturn(rc, rc);
6914
6915	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6916	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6917	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6918
6919	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6920	{
6921	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6922	return VINF_EM_RAW_INTERRUPT;
6923	}
6924
6925	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6926	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6927	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6928	{
6929	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6930	return VINF_SUCCESS;
6931	}
6932	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6933	{
6934	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6935	return rc;
6936	}
6937
6938	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6939	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6940	switch (uIntrType)
6941	{
6942	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6943	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6944	/* no break */
6945	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
6946	{
6947	switch (uVector)
6948	{
6949	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6950	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6951	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6952	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6953	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6954	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
6955	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6956	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6957	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6958	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6959	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6960	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6961	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6962	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6963	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6964	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6965	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6966	#endif
6967	default:
6968	{
6969	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6970	AssertRCReturn(rc, rc);
6971
6972	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6973	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6974	{
6975	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6976	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6977	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6978	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6979	AssertRCReturn(rc, rc);
6980	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6981	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
6982	0 /* GCPtrFaultAddress */);
6983	AssertRCReturn(rc, rc);
6984	}
6985	else
6986	{
6987	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6988	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6989	}
6990	break;
6991	}
6992	}
6993	break;
6994	}
6995
6996	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6997	default:
6998	{
6999	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7000	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7001	break;
7002	}
7003	}
7004	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7005	return rc;
7006	}
7007
7008
7009	/**
7010	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7011	*/
7012	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7013	{
7014	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7015
7016	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7017	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7018	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7019	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7020	AssertRCReturn(rc, rc);
7021
7022	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7023	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7024	return VINF_SUCCESS;
7025	}
7026
7027
7028	/**
7029	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7030	*/
7031	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7032	{
7033	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7034	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7035	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7036	}
7037
7038
7039	/**
7040	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7041	*/
7042	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7043	{
7044	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7045	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7046	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7047	}
7048
7049
7050	/**
7051	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7052	*/
7053	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7054	{
7055	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7056	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7057	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7058	}
7059
7060
7061	/**
7062	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7063	*/
7064	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7065	{
7066	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7067	PVM pVM = pVCpu->CTX_SUFF(pVM);
7068	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7069	if (RT_LIKELY(rc == VINF_SUCCESS))
7070	{
7071	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7072	Assert(pVmxTransient->cbInstr == 2);
7073	}
7074	else
7075	{
7076	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7077	rc = VERR_EM_INTERPRETER;
7078	}
7079	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7080	return rc;
7081	}
7082
7083
7084	/**
7085	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7086	*/
7087	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7088	{
7089	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7090	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7091	AssertRCReturn(rc, rc);
7092
7093	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7094	return VINF_EM_RAW_EMULATE_INSTR;
7095
7096	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7097	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7098	}
7099
7100
7101	/**
7102	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7103	*/
7104	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7105	{
7106	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7107	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7108	AssertRCReturn(rc, rc);
7109
7110	PVM pVM = pVCpu->CTX_SUFF(pVM);
7111	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7112	if (RT_LIKELY(rc == VINF_SUCCESS))
7113	{
7114	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7115	Assert(pVmxTransient->cbInstr == 2);
7116	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7117	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7118	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7119	}
7120	else
7121	{
7122	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7123	rc = VERR_EM_INTERPRETER;
7124	}
7125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7126	return rc;
7127	}
7128
7129
7130	/**
7131	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7132	*/
7133	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7134	{
7135	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7136	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7137	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7138	AssertRCReturn(rc, rc);
7139
7140	PVM pVM = pVCpu->CTX_SUFF(pVM);
7141	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7142	if (RT_LIKELY(rc == VINF_SUCCESS))
7143	{
7144	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7145	Assert(pVmxTransient->cbInstr == 3);
7146	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7147	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7148	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7149	}
7150	else
7151	{
7152	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7153	rc = VERR_EM_INTERPRETER;
7154	}
7155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7156	return rc;
7157	}
7158
7159
7160	/**
7161	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7162	*/
7163	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7164	{
7165	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7166	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7167	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7168	AssertRCReturn(rc, rc);
7169
7170	PVM pVM = pVCpu->CTX_SUFF(pVM);
7171	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7172	if (RT_LIKELY(rc == VINF_SUCCESS))
7173	{
7174	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7175	Assert(pVmxTransient->cbInstr == 2);
7176	}
7177	else
7178	{
7179	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7180	rc = VERR_EM_INTERPRETER;
7181	}
7182	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7183	return rc;
7184	}
7185
7186
7187	/**
7188	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7189	*/
7190	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7191	{
7192	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7193	PVM pVM = pVCpu->CTX_SUFF(pVM);
7194	Assert(!pVM->hm.s.fNestedPaging);
7195
7196	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7197	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7198	AssertRCReturn(rc, rc);
7199
7200	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7201	rc = VBOXSTRICTRC_VAL(rc2);
7202	if (RT_LIKELY(rc == VINF_SUCCESS))
7203	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7204	else
7205	{
7206	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7207	pVmxTransient->uExitQualification, rc));
7208	}
7209	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7210	return rc;
7211	}
7212
7213
7214	/**
7215	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7216	*/
7217	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7218	{
7219	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7220	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7221	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7222	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7223	AssertRCReturn(rc, rc);
7224
7225	PVM pVM = pVCpu->CTX_SUFF(pVM);
7226	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7227	if (RT_LIKELY(rc == VINF_SUCCESS))
7228	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7229	else
7230	{
7231	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7232	rc = VERR_EM_INTERPRETER;
7233	}
7234	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7235	return rc;
7236	}
7237
7238
7239	/**
7240	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7241	*/
7242	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7243	{
7244	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7245	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7246	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7247	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7248	AssertRCReturn(rc, rc);
7249
7250	PVM pVM = pVCpu->CTX_SUFF(pVM);
7251	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7252	rc = VBOXSTRICTRC_VAL(rc2);
7253	if (RT_LIKELY( rc == VINF_SUCCESS
7254	\|\| rc == VINF_EM_HALT))
7255	{
7256	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7257	AssertRCReturn(rc3, rc3);
7258
7259	if ( rc == VINF_EM_HALT
7260	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7261	{
7262	rc = VINF_SUCCESS;
7263	}
7264	}
7265	else
7266	{
7267	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7268	rc = VERR_EM_INTERPRETER;
7269	}
7270	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7271	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7273	return rc;
7274	}
7275
7276
7277	/**
7278	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7279	*/
7280	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7281	{
7282	/*
7283	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7284	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7285	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7286	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7287	*/
7288	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7289	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7290	}
7291
7292
7293	/**
7294	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7295	*/
7296	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7297	{
7298	/*
7299	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7300	* root operation. If we get there there is something funny going on.
7301	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7302	*/
7303	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7304	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7305	}
7306
7307
7308	/**
7309	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7310	*/
7311	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7312	{
7313	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7314	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7315	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7316	}
7317
7318
7319	/**
7320	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7321	*/
7322	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7323	{
7324	/*
7325	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7326	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7327	* See Intel spec. 25.3 "Other Causes of VM-exits".
7328	*/
7329	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7330	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7331	}
7332
7333
7334	/**
7335	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7336	* VM-exit.
7337	*/
7338	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7339	{
7340	/*
7341	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7342	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7343	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7344	*/
7345	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7346	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7347	}
7348
7349
7350	/**
7351	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7352	* VM-exit.
7353	*/
7354	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7355	{
7356	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7357	return VINF_EM_RESET;
7358	}
7359
7360
7361	/**
7362	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7363	*/
7364	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7365	{
7366	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7367	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7368	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7369	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7370	AssertRCReturn(rc, rc);
7371
7372	pMixedCtx->rip++;
7373	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7374	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7375	rc = VINF_SUCCESS;
7376	else
7377	rc = VINF_EM_HALT;
7378
7379	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7380	return rc;
7381	}
7382
7383
7384	/**
7385	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7386	*/
7387	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7388	{
7389	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7390	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7391	return VINF_SUCCESS;
7392	}
7393
7394
7395	/**
7396	* VM-exit handler for expiry of the VMX preemption timer.
7397	*/
7398	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7399	{
7400	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7401
7402	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7403	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7404	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7405
7406	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7407	PVM pVM = pVCpu->CTX_SUFF(pVM);
7408	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7410	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7411	}
7412
7413
7414	/**
7415	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7416	*/
7417	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7418	{
7419	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7420	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7421	/** @todo check if XSETBV is supported by the recompiler. */
7422	return VERR_EM_INTERPRETER;
7423	}
7424
7425
7426	/**
7427	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7428	*/
7429	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7430	{
7431	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7432	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7433	/** @todo implement EMInterpretInvpcid() */
7434	return VERR_EM_INTERPRETER;
7435	}
7436
7437
7438	/**
7439	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7440	* Error VM-exit.
7441	*/
7442	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7443	{
7444	uint32_t uIntrState;
7445	RTHCUINTREG uHCReg;
7446	uint64_t u64Val;
7447	uint32_t u32Val;
7448
7449	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7450	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7451	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7452	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7453	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7454	AssertRCReturn(rc, rc);
7455
7456	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7457	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7458	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7459	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7460
7461	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7462	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7463	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7464	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7465	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7466	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7467	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7468	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7469	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7470	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7471	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7472	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7473
7474	PVM pVM = pVCpu->CTX_SUFF(pVM);
7475	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7476
7477	return VERR_VMX_INVALID_GUEST_STATE;
7478	}
7479
7480
7481	/**
7482	* VM-exit handler for VM-entry failure due to an MSR-load
7483	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7484	*/
7485	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7486	{
7487	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7488	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7489	}
7490
7491
7492	/**
7493	* VM-exit handler for VM-entry failure due to a machine-check event
7494	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7495	*/
7496	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7497	{
7498	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7499	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7500	}
7501
7502
7503	/**
7504	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7505	* theory.
7506	*/
7507	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7508	{
7509	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7510	return VERR_VMX_UNDEFINED_EXIT_CODE;
7511	}
7512
7513
7514	/**
7515	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7516	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7517	* Conditional VM-exit.
7518	*/
7519	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7520	{
7521	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7522	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7523	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7524	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7525	return VERR_EM_INTERPRETER;
7526	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7527	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7528	}
7529
7530
7531	/**
7532	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7533	*/
7534	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7535	{
7536	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7537	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7539	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7540	return VERR_EM_INTERPRETER;
7541	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7542	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7543	}
7544
7545
7546	/**
7547	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7548	*/
7549	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7550	{
7551	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7552	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7553	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7554	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7555	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7556	AssertRCReturn(rc, rc);
7557
7558	PVM pVM = pVCpu->CTX_SUFF(pVM);
7559	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7560	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7561	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7563
7564	if (RT_LIKELY(rc == VINF_SUCCESS))
7565	{
7566	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7567	Assert(pVmxTransient->cbInstr == 2);
7568	}
7569	return rc;
7570	}
7571
7572
7573	/**
7574	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7575	*/
7576	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7577	{
7578	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7579	PVM pVM = pVCpu->CTX_SUFF(pVM);
7580	int rc = VINF_SUCCESS;
7581
7582	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7583	if ( pVM->hm.s.fTPRPatchingActive
7584	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7585	{
7586	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7587	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7588	{
7589	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7590	AssertRC(rc);
7591	}
7592
7593	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7594	Assert(pVmxTransient->cbInstr == 2);
7595	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7596	return VINF_SUCCESS;
7597	}
7598
7599	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7600	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7601	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7602	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7603	AssertRCReturn(rc, rc);
7604	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7605
7606	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7607	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7609
7610	if (RT_LIKELY(rc == VINF_SUCCESS))
7611	{
7612	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7613
7614	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7615	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7616	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7617	{
7618	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7619	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7620	}
7621	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7622	{
7623	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7624	AssertRCReturn(rc, rc);
7625	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7626	}
7627
7628	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7629	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7630	{
7631	switch (pMixedCtx->ecx)
7632	{
7633	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7634	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7635	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7636	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7637	case MSR_K8_FS_BASE: /* no break */
7638	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7639	}
7640	}
7641	#ifdef VBOX_STRICT
7642	else
7643	{
7644	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7645	switch (pMixedCtx->ecx)
7646	{
7647	case MSR_IA32_SYSENTER_CS:
7648	case MSR_IA32_SYSENTER_EIP:
7649	case MSR_IA32_SYSENTER_ESP:
7650	case MSR_K8_FS_BASE:
7651	case MSR_K8_GS_BASE:
7652	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7653	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7654
7655	case MSR_K8_LSTAR:
7656	case MSR_K6_STAR:
7657	case MSR_K8_SF_MASK:
7658	case MSR_K8_TSC_AUX:
7659	{
7660	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7661	pMixedCtx->ecx));
7662	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7663	}
7664
7665	case MSR_K8_KERNEL_GS_BASE:
7666	{
7667	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7668	pMixedCtx->ecx));
7669	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7670	}
7671	}
7672	}
7673	#endif /* VBOX_STRICT */
7674	}
7675	return rc;
7676	}
7677
7678
7679	/**
7680	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7681	*/
7682	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7683	{
7684	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7685	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7686	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7687	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7688	return VERR_EM_INTERPRETER;
7689	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7690	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7691	}
7692
7693
7694	/**
7695	* VM-exit handler for when the TPR value is lowered below the specified
7696	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7697	*/
7698	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7699	{
7700	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7701	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7702
7703	/*
7704	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7705	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7706	* resume guest execution.
7707	*/
7708	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7710	return VINF_SUCCESS;
7711	}
7712
7713
7714	/**
7715	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7716	* VM-exit.
7717	*
7718	* @retval VINF_SUCCESS when guest execution can continue.
7719	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7720	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7721	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7722	* recompiler.
7723	*/
7724	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7725	{
7726	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7727	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7728	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7729	AssertRCReturn(rc, rc);
7730
7731	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7732	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7733	PVM pVM = pVCpu->CTX_SUFF(pVM);
7734	switch (uAccessType)
7735	{
7736	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7737	{
7738	#if 0
7739	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7740	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7741	#else
7742	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7743	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7744	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7745	#endif
7746	AssertRCReturn(rc, rc);
7747
7748	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7749	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7750	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7751	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7752
7753	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7754	{
7755	case 0: /* CR0 */
7756	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7757	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7758	break;
7759	case 2: /* C2 **/
7760	/* Nothing to do here, CR2 it's not part of the VMCS. */
7761	break;
7762	case 3: /* CR3 */
7763	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7764	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7765	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7766	break;
7767	case 4: /* CR4 */
7768	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7769	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7770	break;
7771	case 8: /* CR8 */
7772	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7773	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7774	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7775	break;
7776	default:
7777	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7778	break;
7779	}
7780
7781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7782	break;
7783	}
7784
7785	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7786	{
7787	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7788	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7789	AssertRCReturn(rc, rc);
7790	Assert( !pVM->hm.s.fNestedPaging
7791	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7792	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7793
7794	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7795	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7796	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7797
7798	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7799	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7800	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7801	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7802	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7803	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7804	break;
7805	}
7806
7807	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7808	{
7809	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7810	AssertRCReturn(rc, rc);
7811	rc = EMInterpretCLTS(pVM, pVCpu);
7812	AssertRCReturn(rc, rc);
7813	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7814	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7815	Log(("CRX CLTS write rc=%d\n", rc));
7816	break;
7817	}
7818
7819	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7820	{
7821	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7822	AssertRCReturn(rc, rc);
7823	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7824	if (RT_LIKELY(rc == VINF_SUCCESS))
7825	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7826	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7827	Log(("CRX LMSW write rc=%d\n", rc));
7828	break;
7829	}
7830
7831	default:
7832	{
7833	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7834	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7835	}
7836	}
7837
7838	/* Validate possible error codes. */
7839	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7840	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7841	if (RT_SUCCESS(rc))
7842	{
7843	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7844	AssertRCReturn(rc2, rc2);
7845	}
7846
7847	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7848	return rc;
7849	}
7850
7851
7852	/**
7853	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7854	* VM-exit.
7855	*/
7856	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7857	{
7858	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7859	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7860
7861	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7862	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7863	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7864	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7865	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7866	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7867	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7868	AssertRCReturn(rc, rc);
7869	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7870
7871	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7872	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7873	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7874	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7875	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7876	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7877	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7878
7879	/* I/O operation lookup arrays. */
7880	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7881	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7882
7883	const uint32_t cbSize = s_aIOSize[uIOWidth];
7884	const uint32_t cbInstr = pVmxTransient->cbInstr;
7885	PVM pVM = pVCpu->CTX_SUFF(pVM);
7886	if (fIOString)
7887	{
7888	/* INS/OUTS - I/O String instruction. */
7889	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7890	/** @todo for now manually disassemble later optimize by getting the fields from
7891	* the VMCS. */
7892	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7893	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7894	* segment prefix info. */
7895	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7896	if (RT_SUCCESS(rc))
7897	{
7898	if (fIOWrite)
7899	{
7900	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7901	(DISCPUMODE)pDis->uAddrMode, cbSize);
7902	rc = VBOXSTRICTRC_VAL(rc2);
7903	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7904	}
7905	else
7906	{
7907	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7908	(DISCPUMODE)pDis->uAddrMode, cbSize);
7909	rc = VBOXSTRICTRC_VAL(rc2);
7910	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7911	}
7912	}
7913	else
7914	{
7915	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7916	rc = VINF_EM_RAW_EMULATE_INSTR;
7917	}
7918	}
7919	else
7920	{
7921	/* IN/OUT - I/O instruction. */
7922	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7923	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7924	if (fIOWrite)
7925	{
7926	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7927	rc = VBOXSTRICTRC_VAL(rc2);
7928	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7929	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7931	}
7932	else
7933	{
7934	uint32_t u32Result = 0;
7935	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7936	rc = VBOXSTRICTRC_VAL(rc2);
7937	if (IOM_SUCCESS(rc))
7938	{
7939	/* Save result of I/O IN instr. in AL/AX/EAX. */
7940	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7941	}
7942	else if (rc == VINF_IOM_R3_IOPORT_READ)
7943	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7944	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7945	}
7946	}
7947
7948	if (IOM_SUCCESS(rc))
7949	{
7950	pMixedCtx->rip += cbInstr;
7951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7952	if (RT_LIKELY(rc == VINF_SUCCESS))
7953	{
7954	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7955	AssertRCReturn(rc, rc);
7956
7957	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7958	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7959	{
7960	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7961	for (unsigned i = 0; i < 4; i++)
7962	{
7963	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7964	if ( ( uIOPort >= pMixedCtx->dr[i]
7965	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7966	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7967	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7968	{
7969	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7970	uint64_t uDR6 = ASMGetDR6();
7971
7972	/* Clear all breakpoint status flags and set the one we just hit. */
7973	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7974	uDR6 \|= (uint64_t)RT_BIT(i);
7975
7976	/*
7977	* Note: AMD64 Architecture Programmer's Manual 13.1:
7978	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7979	* be cleared by software after the contents have been read.
7980	*/
7981	ASMSetDR6(uDR6);
7982
7983	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7984	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7985
7986	/* Paranoia. */
7987	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7988	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7989	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7990
7991	/* Resync DR7 */
7992	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7993	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7994
7995	/* Set #DB to be injected into the VM and continue guest execution. */
7996	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
7997	break;
7998	}
7999	}
8000	}
8001	}
8002	}
8003
8004	#ifdef DEBUG
8005	if (rc == VINF_IOM_R3_IOPORT_READ)
8006	Assert(!fIOWrite);
8007	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8008	Assert(fIOWrite);
8009	else
8010	{
8011	AssertMsg( RT_FAILURE(rc)
8012	\|\| rc == VINF_SUCCESS
8013	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8014	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8015	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8016	}
8017	#endif
8018
8019	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8020	return rc;
8021	}
8022
8023
8024	/**
8025	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8026	* VM-exit.
8027	*/
8028	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8029	{
8030	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8031
8032	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8033	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8034	AssertRCReturn(rc, rc);
8035	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8036	{
8037	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8038	AssertRCReturn(rc, rc);
8039	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8040	{
8041	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8042	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8043	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8044	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8045	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8046	{
8047	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8048	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8049
8050	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8051	Assert(!pVCpu->hm.s.Event.fPending);
8052	pVCpu->hm.s.Event.fPending = true;
8053	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8054	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8055	AssertRCReturn(rc, rc);
8056	if (fErrorCodeValid)
8057	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8058	else
8059	pVCpu->hm.s.Event.u32ErrCode = 0;
8060	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8061	&& uVector == X86_XCPT_PF)
8062	{
8063	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8064	}
8065	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8066	}
8067	}
8068	}
8069	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8070	* emulation. */
8071	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8072	return VERR_EM_INTERPRETER;
8073	}
8074
8075
8076	/**
8077	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8078	*/
8079	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8080	{
8081	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8082	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8083	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8084	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8085	AssertRCReturn(rc, rc);
8086	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8087	return VINF_EM_DBG_STOP;
8088	}
8089
8090
8091	/**
8092	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8093	*/
8094	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8095	{
8096	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8097
8098	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8099	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8100	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8101	return VINF_SUCCESS;
8102	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8103	return rc;
8104
8105	#if 0
8106	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8107	* just sync the whole thing. */
8108	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8109	#else
8110	/* Aggressive state sync. for now. */
8111	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8112	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8113	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8114	#endif
8115	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8116	AssertRCReturn(rc, rc);
8117
8118	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8119	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8120	switch (uAccessType)
8121	{
8122	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8123	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8124	{
8125	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8126	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8127	{
8128	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8129	}
8130
8131	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8132	GCPhys &= PAGE_BASE_GC_MASK;
8133	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8134	PVM pVM = pVCpu->CTX_SUFF(pVM);
8135	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8136	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8137
8138	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8139	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8140	CPUMCTX2CORE(pMixedCtx), GCPhys);
8141	rc = VBOXSTRICTRC_VAL(rc2);
8142	Log(("ApicAccess rc=%d\n", rc));
8143	if ( rc == VINF_SUCCESS
8144	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8145	\|\| rc == VERR_PAGE_NOT_PRESENT)
8146	{
8147	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8148	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8149	rc = VINF_SUCCESS;
8150	}
8151	break;
8152	}
8153
8154	default:
8155	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8156	rc = VINF_EM_RAW_EMULATE_INSTR;
8157	break;
8158	}
8159
8160	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8161	return rc;
8162	}
8163
8164
8165	/**
8166	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8167	* VM-exit.
8168	*/
8169	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8170	{
8171	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8172
8173	/* We should -not- get this VM-exit if the guest is debugging. */
8174	if (CPUMIsGuestDebugStateActive(pVCpu))
8175	{
8176	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8177	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8178	}
8179
8180	int rc = VERR_INTERNAL_ERROR_5;
8181	if ( !DBGFIsStepping(pVCpu)
8182	&& !CPUMIsHyperDebugStateActive(pVCpu))
8183	{
8184	/* Don't intercept MOV DRx. */
8185	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8186	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8187	AssertRCReturn(rc, rc);
8188
8189	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8190	PVM pVM = pVCpu->CTX_SUFF(pVM);
8191	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8192	AssertRC(rc);
8193	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8194
8195	#ifdef VBOX_WITH_STATISTICS
8196	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8197	AssertRCReturn(rc, rc);
8198	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8199	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8200	else
8201	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8202	#endif
8203	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8204	return VINF_SUCCESS;
8205	}
8206
8207	/*
8208	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8209	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8210	*/
8211	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8212	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8213	AssertRCReturn(rc, rc);
8214
8215	PVM pVM = pVCpu->CTX_SUFF(pVM);
8216	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8217	{
8218	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8219	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8220	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8221	if (RT_SUCCESS(rc))
8222	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8223	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8224	}
8225	else
8226	{
8227	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8228	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8229	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8230	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8231	}
8232
8233	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8234	if (RT_SUCCESS(rc))
8235	{
8236	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8237	AssertRCReturn(rc2, rc2);
8238	}
8239	return rc;
8240	}
8241
8242
8243	/**
8244	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8245	* Conditional VM-exit.
8246	*/
8247	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8248	{
8249	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8250	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8251
8252	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8253	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8254	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8255	return VINF_SUCCESS;
8256	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8257	return rc;
8258
8259	RTGCPHYS GCPhys = 0;
8260	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8261
8262	#if 0
8263	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8264	#else
8265	/* Aggressive state sync. for now. */
8266	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8267	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8268	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8269	#endif
8270	AssertRCReturn(rc, rc);
8271
8272	/*
8273	* If we succeed, resume guest execution.
8274	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8275	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8276	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8277	* weird case. See @bugref{6043}.
8278	*/
8279	PVM pVM = pVCpu->CTX_SUFF(pVM);
8280	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8281	rc = VBOXSTRICTRC_VAL(rc2);
8282	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8283	if ( rc == VINF_SUCCESS
8284	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8285	\|\| rc == VERR_PAGE_NOT_PRESENT)
8286	{
8287	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8288	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8289	return VINF_SUCCESS;
8290	}
8291	return rc;
8292	}
8293
8294
8295	/**
8296	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8297	* VM-exit.
8298	*/
8299	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8300	{
8301	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8302	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8303
8304	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8305	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8306	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8307	return VINF_SUCCESS;
8308	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8309	return rc;
8310
8311	RTGCPHYS GCPhys = 0;
8312	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8313	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8314	#if 0
8315	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8316	#else
8317	/* Aggressive state sync. for now. */
8318	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8319	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8320	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8321	#endif
8322	AssertRCReturn(rc, rc);
8323
8324	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8325	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8326
8327	RTGCUINT uErrorCode = 0;
8328	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8329	uErrorCode \|= X86_TRAP_PF_ID;
8330	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8331	uErrorCode \|= X86_TRAP_PF_RW;
8332	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8333	uErrorCode \|= X86_TRAP_PF_P;
8334
8335	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8336
8337	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8338	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8339
8340	/* Handle the pagefault trap for the nested shadow table. */
8341	PVM pVM = pVCpu->CTX_SUFF(pVM);
8342	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8343	TRPMResetTrap(pVCpu);
8344
8345	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8346	if ( rc == VINF_SUCCESS
8347	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8348	\|\| rc == VERR_PAGE_NOT_PRESENT)
8349	{
8350	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8352	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8353	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8354	return VINF_SUCCESS;
8355	}
8356
8357	Log(("EPT return to ring-3 rc=%d\n"));
8358	return rc;
8359	}
8360
8361
8362	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8363	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8364	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8365	/**
8366	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8367	*/
8368	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8369	{
8370	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8372
8373	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8374	AssertRCReturn(rc, rc);
8375
8376	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8377	{
8378	/* Old-style FPU error reporting needs some extra work. */
8379	/** @todo don't fall back to the recompiler, but do it manually. */
8380	return VERR_EM_INTERPRETER;
8381	}
8382	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8383	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8384	return rc;
8385	}
8386
8387
8388	/**
8389	* VM-exit exception handler for #BP (Breakpoint exception).
8390	*/
8391	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8392	{
8393	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8394	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8395
8396	/** @todo Try optimize this by not saving the entire guest state unless
8397	* really needed. */
8398	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8399	AssertRCReturn(rc, rc);
8400
8401	PVM pVM = pVCpu->CTX_SUFF(pVM);
8402	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8403	if (rc == VINF_EM_RAW_GUEST_TRAP)
8404	{
8405	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8406	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8407	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8408	AssertRCReturn(rc, rc);
8409
8410	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8411	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8412	}
8413
8414	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8415	return rc;
8416	}
8417
8418
8419	/**
8420	* VM-exit exception handler for #DB (Debug exception).
8421	*/
8422	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8423	{
8424	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8425	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8426
8427	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8428	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8429	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8430	AssertRCReturn(rc, rc);
8431
8432	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8433	uint64_t uDR6 = X86_DR6_INIT_VAL;
8434	uDR6 \|= (pVmxTransient->uExitQualification
8435	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8436	PVM pVM = pVCpu->CTX_SUFF(pVM);
8437	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8438	if (rc == VINF_EM_RAW_GUEST_TRAP)
8439	{
8440	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8441	pMixedCtx->dr[6] = uDR6;
8442
8443	if (CPUMIsGuestDebugStateActive(pVCpu))
8444	ASMSetDR6(pMixedCtx->dr[6]);
8445
8446	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8447
8448	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8449	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8450
8451	/* Paranoia. */
8452	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8453	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8454	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8455
8456	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8457	AssertRCReturn(rc,rc);
8458
8459	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8460	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8461	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8462	AssertRCReturn(rc2, rc2);
8463	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8464	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8465	rc = VINF_SUCCESS;
8466	}
8467
8468	return rc;
8469	}
8470
8471
8472	/**
8473	* VM-exit exception handler for #NM (Device-not-available exception: floating
8474	* point exception).
8475	*/
8476	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8477	{
8478	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8479
8480	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8481	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8482	#endif
8483
8484	/* We require CR0 and EFER. EFER is always up-to-date. */
8485	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8486	AssertRCReturn(rc, rc);
8487
8488	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8489	PVM pVM = pVCpu->CTX_SUFF(pVM);
8490	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8491	if (rc == VINF_SUCCESS)
8492	{
8493	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8494	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8495	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8496	return VINF_SUCCESS;
8497	}
8498
8499	/* Forward #NM to the guest. */
8500	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8501	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8502	AssertRCReturn(rc, rc);
8503	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8504	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8506	return rc;
8507	}
8508
8509
8510	/**
8511	* VM-exit exception handler for #GP (General-protection exception).
8512	*
8513	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8514	*/
8515	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8516	{
8517	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8518	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8519
8520	int rc = VERR_INTERNAL_ERROR_5;
8521	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8522	{
8523	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8524	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8525	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8526	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8527	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8528	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8529	AssertRCReturn(rc, rc);
8530	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8531	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8532	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8533	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8534	return rc;
8535	#else
8536	/* We don't intercept #GP. */
8537	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8538	return VERR_VMX_UNEXPECTED_EXCEPTION;
8539	#endif
8540	}
8541
8542	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8543	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8544
8545	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8546	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8547	AssertRCReturn(rc, rc);
8548
8549	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8550	uint32_t cbOp = 0;
8551	PVM pVM = pVCpu->CTX_SUFF(pVM);
8552	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8553	if (RT_SUCCESS(rc))
8554	{
8555	rc = VINF_SUCCESS;
8556	Assert(cbOp == pDis->cbInstr);
8557	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8558	switch (pDis->pCurInstr->uOpcode)
8559	{
8560	case OP_CLI:
8561	pMixedCtx->eflags.Bits.u1IF = 0;
8562	pMixedCtx->rip += pDis->cbInstr;
8563	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8564	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8565	break;
8566
8567	case OP_STI:
8568	pMixedCtx->eflags.Bits.u1IF = 1;
8569	pMixedCtx->rip += pDis->cbInstr;
8570	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8571	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8572	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8573	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8574	break;
8575
8576	case OP_HLT:
8577	rc = VINF_EM_HALT;
8578	pMixedCtx->rip += pDis->cbInstr;
8579	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8581	break;
8582
8583	case OP_POPF:
8584	{
8585	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8586	uint32_t cbParm = 0;
8587	uint32_t uMask = 0;
8588	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8589	{
8590	cbParm = 4;
8591	uMask = 0xffffffff;
8592	}
8593	else
8594	{
8595	cbParm = 2;
8596	uMask = 0xffff;
8597	}
8598
8599	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8600	RTGCPTR GCPtrStack = 0;
8601	X86EFLAGS uEflags;
8602	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8603	&GCPtrStack);
8604	if (RT_SUCCESS(rc))
8605	{
8606	Assert(sizeof(uEflags.u32) >= cbParm);
8607	uEflags.u32 = 0;
8608	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8609	}
8610	if (RT_FAILURE(rc))
8611	{
8612	rc = VERR_EM_INTERPRETER;
8613	break;
8614	}
8615	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8616	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8617	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8618	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8619	pMixedCtx->eflags.Bits.u1RF = 0;
8620	pMixedCtx->esp += cbParm;
8621	pMixedCtx->esp &= uMask;
8622	pMixedCtx->rip += pDis->cbInstr;
8623	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8624	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8625	break;
8626	}
8627
8628	case OP_PUSHF:
8629	{
8630	uint32_t cbParm = 0;
8631	uint32_t uMask = 0;
8632	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8633	{
8634	cbParm = 4;
8635	uMask = 0xffffffff;
8636	}
8637	else
8638	{
8639	cbParm = 2;
8640	uMask = 0xffff;
8641	}
8642
8643	/* Get the stack pointer & push the contents of eflags onto the stack. */
8644	RTGCPTR GCPtrStack = 0;
8645	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8646	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8647	if (RT_FAILURE(rc))
8648	{
8649	rc = VERR_EM_INTERPRETER;
8650	break;
8651	}
8652	X86EFLAGS uEflags;
8653	uEflags = pMixedCtx->eflags;
8654	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8655	uEflags.Bits.u1RF = 0;
8656	uEflags.Bits.u1VM = 0;
8657
8658	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8659	if (RT_FAILURE(rc))
8660	{
8661	rc = VERR_EM_INTERPRETER;
8662	break;
8663	}
8664	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8665	pMixedCtx->esp -= cbParm;
8666	pMixedCtx->esp &= uMask;
8667	pMixedCtx->rip += pDis->cbInstr;
8668	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8669	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8670	break;
8671	}
8672
8673	case OP_IRET:
8674	{
8675	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8676	* instruction reference. */
8677	RTGCPTR GCPtrStack = 0;
8678	uint32_t uMask = 0xffff;
8679	uint16_t aIretFrame[3];
8680	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8681	{
8682	rc = VERR_EM_INTERPRETER;
8683	break;
8684	}
8685	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8686	&GCPtrStack);
8687	if (RT_SUCCESS(rc))
8688	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8689	if (RT_FAILURE(rc))
8690	{
8691	rc = VERR_EM_INTERPRETER;
8692	break;
8693	}
8694	pMixedCtx->eip = 0;
8695	pMixedCtx->ip = aIretFrame[0];
8696	pMixedCtx->cs.Sel = aIretFrame[1];
8697	pMixedCtx->cs.ValidSel = aIretFrame[1];
8698	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8699	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8700	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8701	pMixedCtx->sp += sizeof(aIretFrame);
8702	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8703	\| HM_CHANGED_GUEST_RFLAGS;
8704	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8705	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8706	break;
8707	}
8708
8709	case OP_INT:
8710	{
8711	uint16_t uVector = pDis->Param1.uValue & 0xff;
8712	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8713	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8714	break;
8715	}
8716
8717	case OP_INTO:
8718	{
8719	if (pMixedCtx->eflags.Bits.u1OF)
8720	{
8721	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8722	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8723	}
8724	break;
8725	}
8726
8727	default:
8728	{
8729	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8730	EMCODETYPE_SUPERVISOR);
8731	rc = VBOXSTRICTRC_VAL(rc2);
8732	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8733	Log(("#GP rc=%Rrc\n", rc));
8734	break;
8735	}
8736	}
8737	}
8738	else
8739	rc = VERR_EM_INTERPRETER;
8740
8741	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8742	("#GP Unexpected rc=%Rrc\n", rc));
8743	return rc;
8744	}
8745
8746
8747	/**
8748	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8749	* the exception reported in the VMX transient structure back into the VM.
8750	*
8751	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8752	* VMX transient structure to be up-to-date.
8753	*/
8754	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8755	{
8756	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8757
8758	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8759	hmR0VmxCheckExitDueToEventDelivery(). */
8760	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8761	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8762	AssertRCReturn(rc, rc);
8763	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8764	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8765	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8766	return VINF_SUCCESS;
8767	}
8768
8769
8770	/**
8771	* VM-exit exception handler for #PF (Page-fault exception).
8772	*/
8773	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8774	{
8775	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8776	PVM pVM = pVCpu->CTX_SUFF(pVM);
8777	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8778	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8779	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8780	AssertRCReturn(rc, rc);
8781
8782	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8783	if (pVM->hm.s.fNestedPaging)
8784	{
8785	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8786	{
8787	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8788	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8789	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8790	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8791	}
8792	else
8793	{
8794	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8795	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8796	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8797	Log(("Pending #DF due to vectoring #PF. NP\n"));
8798	}
8799	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8800	return rc;
8801	}
8802	#else
8803	Assert(!pVM->hm.s.fNestedPaging);
8804	#endif
8805
8806	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8807	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8808	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8809	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8810	AssertRCReturn(rc, rc);
8811	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8812	if ( pVM->hm.s.fTRPPatchingAllowed
8813	&& pVM->hm.s.pGuestPatchMem
8814	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8815	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8816	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8817	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8818	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8819	{
8820	RTGCPHYS GCPhys;
8821	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8822	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8823	if ( rc == VINF_SUCCESS
8824	&& GCPhys == GCPhysApicBase)
8825	{
8826	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8827	AssertRCReturn(rc, rc);
8828
8829	/* Only attempt to patch the instruction once. */
8830	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8831	if (!pPatch)
8832	return VINF_EM_HM_PATCH_TPR_INSTR;
8833	}
8834	}
8835	#endif
8836
8837	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8838	AssertRCReturn(rc, rc);
8839
8840	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8841	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8842
8843	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8844	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8845	(RTGCPTR)pVmxTransient->uExitQualification);
8846
8847	Log(("#PF: rc=%Rrc\n", rc));
8848	if (rc == VINF_SUCCESS)
8849	{
8850	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8851	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8852	* memory? We don't update the whole state here... */
8853	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8854	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8855	TRPMResetTrap(pVCpu);
8856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8857	return rc;
8858	}
8859	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8860	{
8861	if (!pVmxTransient->fVectoringPF)
8862	{
8863	/* It's a guest page fault and needs to be reflected to the guest. */
8864	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8865	TRPMResetTrap(pVCpu);
8866	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8867	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8868	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8869	}
8870	else
8871	{
8872	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8873	TRPMResetTrap(pVCpu);
8874	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8875	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8876	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8877	}
8878
8879	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8880	return VINF_SUCCESS;
8881	}
8882
8883	TRPMResetTrap(pVCpu);
8884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8885	return rc;
8886	}
8887
8888

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45685

Download in other formats: