HMVMXR0.cpp@ 45655

Last change on this file since 45655 was 45655, checked in by vboxsync, 12 years ago

VMMR0/HM: Profiling longjmp round trips are less useful, adjusted STAM counters to reflect true time spent on VMXR0RunGuestCode.

Removed excessive inlining in HMVMXR0.
Added exit-to-ring3 stat to HWVMXR0.

Property svn:eol-style set to native
Property svn:keywords set to Author Date Id Revision

File size: 377.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45655 2013-04-20 14:27:02Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_TRAP_ALL_EXCEPTIONS
41	#endif
42
43
44	/*******************************************************************************
45	* Defined Constants And Macros *
46	*******************************************************************************/
47	#if defined(RT_ARCH_AMD64)
48	# define HMVMX_IS_64BIT_HOST_MODE() (true)
49	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
50	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
51	#else
52	# define HMVMX_IS_64BIT_HOST_MODE() (false)
53	#endif
54
55	/** Use the function table. */
56	#define HMVMX_USE_FUNCTION_TABLE
57
58	/** This bit indicates the segment selector is unusable in VT-x. */
59	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
60
61	/** Determine which tagged-TLB flush handler to use. */
62	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
64	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
65	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
66
67	/**
68	* Updated-guest-state flags.
69	*/
70	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
82	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
83	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
86	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
87	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
88	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
89	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91	\| HMVMX_UPDATED_GUEST_RSP \
92	\| HMVMX_UPDATED_GUEST_RFLAGS \
93	\| HMVMX_UPDATED_GUEST_CR0 \
94	\| HMVMX_UPDATED_GUEST_CR3 \
95	\| HMVMX_UPDATED_GUEST_CR4 \
96	\| HMVMX_UPDATED_GUEST_GDTR \
97	\| HMVMX_UPDATED_GUEST_IDTR \
98	\| HMVMX_UPDATED_GUEST_LDTR \
99	\| HMVMX_UPDATED_GUEST_TR \
100	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101	\| HMVMX_UPDATED_GUEST_DEBUG \
102	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
103	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
106	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
107	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
108	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
109	\| HMVMX_UPDATED_GUEST_APIC_STATE)
110
111	/**
112	* Flags to skip redundant reads of some common VMCS fields that are not part of
113	* the guest-CPU state but are in the transient structure.
114	*/
115	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
116	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
117	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
120	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
121
122	/**
123	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
124	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
125	* we have Nested Paging support.
126	*/
127	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
128	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
129	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
130	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
131	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
132	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
133	\| RT_BIT(X86_XCPT_XF))
134
135	/**
136	* Exception bitmap mask for all contributory exceptions.
137	*/
138	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
139	\| RT_BIT(X86_XCPT_DE))
140
141	/** Maximum VM-instruction error number. */
142	#define HMVMX_INSTR_ERROR_MAX 28
143
144
145	/*******************************************************************************
146	* Structures and Typedefs *
147	*******************************************************************************/
148	/**
149	* A state structure for holding miscellaneous information across
150	* VMX non-root operation and restored after the transition.
151	*/
152	typedef struct VMXTRANSIENT
153	{
154	/** The host's rflags/eflags. */
155	RTCCUINTREG uEFlags;
156	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
157	uint64_t u64LStarMsr;
158	/** The guest's TPR value used for TPR shadowing. */
159	uint8_t u8GuestTpr;
160
161	/** The basic VM-exit reason. */
162	uint16_t uExitReason;
163	/** The VM-exit exit qualification. */
164	RTGCUINTPTR uExitQualification;
165	/** The VM-exit interruption error code. */
166	uint32_t uExitIntrErrorCode;
167
168	/** The VM-exit interruption-information field. */
169	uint32_t uExitIntrInfo;
170	/** Whether the VM-entry failed or not. */
171	bool fVMEntryFailed;
172	/** The VM-exit instruction-length field. */
173	uint32_t cbInstr;
174
175	/** The VM-entry interruption-information field. */
176	uint32_t uEntryIntrInfo;
177	/** The VM-entry exception error code field. */
178	uint32_t uEntryXcptErrorCode;
179	/** The VM-entry instruction length field. */
180	uint32_t cbEntryInstr;
181
182	/** IDT-vectoring information field. */
183	uint32_t uIdtVectoringInfo;
184	/** IDT-vectoring error code. */
185	uint32_t uIdtVectoringErrorCode;
186
187	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
188	uint32_t fVmcsFieldsRead;
189	/** Whether TSC-offsetting should be setup before VM-entry. */
190	bool fUpdateTscOffsettingAndPreemptTimer;
191	/** Whether the VM-exit was caused by a page-fault during delivery of a
192	* contributary exception or a page-fault. */
193	bool fVectoringPF;
194	} VMXTRANSIENT, *PVMXTRANSIENT;
195
196	/**
197	* MSR-bitmap read permissions.
198	*/
199	typedef enum VMXMSREXITREAD
200	{
201	/** Reading this MSR causes a VM-exit. */
202	VMXMSREXIT_INTERCEPT_READ = 0xb,
203	/** Reading this MSR does not cause a VM-exit. */
204	VMXMSREXIT_PASSTHRU_READ
205	} VMXMSREXITREAD;
206
207	/**
208	* MSR-bitmap write permissions.
209	*/
210	typedef enum VMXMSREXITWRITE
211	{
212	/** Writing to this MSR causes a VM-exit. */
213	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
214	/** Writing to this MSR does not cause a VM-exit. */
215	VMXMSREXIT_PASSTHRU_WRITE
216	} VMXMSREXITWRITE;
217
218
219	/*******************************************************************************
220	* Internal Functions *
221	*******************************************************************************/
222	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
223	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
224	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
225	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
226	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
227	#endif
228	#ifndef HMVMX_USE_FUNCTION_TABLE
229	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
230	#endif
231
232	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276
277	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284
285
286	/*******************************************************************************
287	* Global Variables *
288	*******************************************************************************/
289	#ifdef HMVMX_USE_FUNCTION_TABLE
290	/**
291	* VM-exit handler.
292	*
293	* @returns VBox status code.
294	* @param pVCpu Pointer to the VMCPU.
295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
296	* out-of-sync. Make sure to update the required
297	* fields before using them.
298	* @param pVmxTransient Pointer to the VMX-transient structure.
299	*/
300	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	/** Pointer to VM-exit handler. */
302	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
303
304	/**
305	* VMX_EXIT dispatch table.
306	*/
307	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
308	{
309	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
310	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
311	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
312	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
313	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
314	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
315	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
316	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
317	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
318	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
319	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
320	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
321	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
322	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
323	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
324	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
325	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
326	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
327	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
328	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
329	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
330	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
331	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
332	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
333	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
334	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
335	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
336	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
337	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
338	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
339	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
340	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
341	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
342	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
343	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
344	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
345	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
346	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
347	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
348	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
349	/* 40 UNDEFINED */ hmR0VmxExitPause,
350	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
351	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
352	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
353	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
354	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
355	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
356	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
357	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
358	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
359	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
360	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
361	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
362	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
363	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
364	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
365	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
366	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
367	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
368	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
369	};
370	#endif /* HMVMX_USE_FUNCTION_TABLE */
371
372	#ifdef VBOX_STRICT
373	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
374	{
375	/* 0 */ "(Not Used)",
376	/* 1 */ "VMCALL executed in VMX root operation.",
377	/* 2 */ "VMCLEAR with invalid physical address.",
378	/* 3 */ "VMCLEAR with VMXON pointer.",
379	/* 4 */ "VMLAUNCH with non-clear VMCS.",
380	/* 5 */ "VMRESUME with non-launched VMCS.",
381	/* 6 */ "VMRESUME after VMXOFF",
382	/* 7 */ "VM entry with invalid control fields.",
383	/* 8 */ "VM entry with invalid host state fields.",
384	/* 9 */ "VMPTRLD with invalid physical address.",
385	/* 10 */ "VMPTRLD with VMXON pointer.",
386	/* 11 */ "VMPTRLD with incorrect revision identifier.",
387	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
388	/* 13 */ "VMWRITE to read-only VMCS component.",
389	/* 14 */ "(Not Used)",
390	/* 15 */ "VMXON executed in VMX root operation.",
391	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
392	/* 17 */ "VM entry with non-launched executing VMCS.",
393	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
394	/* 19 */ "VMCALL with non-clear VMCS.",
395	/* 20 */ "VMCALL with invalid VM-exit control fields.",
396	/* 21 */ "(Not Used)",
397	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
398	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
399	/* 24 */ "VMCALL with invalid SMM-monitor features.",
400	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
401	/* 26 */ "VM entry with events blocked by MOV SS.",
402	/* 27 */ "(Not Used)",
403	/* 28 */ "Invalid operand to INVEPT/INVVPID."
404	};
405	#endif /* VBOX_STRICT */
406
407
408
409	/**
410	* Updates the VM's last error record. If there was a VMX instruction error,
411	* reads the error data from the VMCS and updates VCPU's last error record as
412	* well.
413	*
414	* @param pVM Pointer to the VM.
415	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
416	* VERR_VMX_UNABLE_TO_START_VM or
417	* VERR_VMX_INVALID_VMCS_FIELD).
418	* @param rc The error code.
419	*/
420	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
421	{
422	AssertPtr(pVM);
423	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
424	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
425	{
426	AssertPtrReturnVoid(pVCpu);
427	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
428	}
429	pVM->hm.s.lLastError = rc;
430	}
431
432
433	/**
434	* Reads the VM-entry interruption-information field from the VMCS into the VMX
435	* transient structure.
436	*
437	* @returns VBox status code.
438	* @param pVmxTransient Pointer to the VMX transient structure.
439	*
440	* @remarks No-long-jump zone!!!
441	*/
442	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
443	{
444	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
445	AssertRCReturn(rc, rc);
446	return VINF_SUCCESS;
447	}
448
449
450	/**
451	* Reads the VM-entry exception error code field from the VMCS into
452	* the VMX transient structure.
453	*
454	* @returns VBox status code.
455	* @param pVmxTransient Pointer to the VMX transient structure.
456	*
457	* @remarks No-long-jump zone!!!
458	*/
459	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
460	{
461	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
462	AssertRCReturn(rc, rc);
463	return VINF_SUCCESS;
464	}
465
466
467	/**
468	* Reads the VM-entry exception error code field from the VMCS into
469	* the VMX transient structure.
470	*
471	* @returns VBox status code.
472	* @param pVCpu Pointer to the VMCPU.
473	* @param pVmxTransient Pointer to the VMX transient structure.
474	*
475	* @remarks No-long-jump zone!!!
476	*/
477	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
478	{
479	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
480	AssertRCReturn(rc, rc);
481	return VINF_SUCCESS;
482	}
483
484
485	/**
486	* Reads the VM-exit interruption-information field from the VMCS into the VMX
487	* transient structure.
488	*
489	* @returns VBox status code.
490	* @param pVCpu Pointer to the VMCPU.
491	* @param pVmxTransient Pointer to the VMX transient structure.
492	*/
493	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
494	{
495	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
496	{
497	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
498	AssertRCReturn(rc, rc);
499	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
500	}
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Reads the VM-exit interruption error code from the VMCS into the VMX
507	* transient structure.
508	*
509	* @returns VBox status code.
510	* @param pVCpu Pointer to the VMCPU.
511	* @param pVmxTransient Pointer to the VMX transient structure.
512	*/
513	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
514	{
515	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
516	{
517	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
518	AssertRCReturn(rc, rc);
519	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
520	}
521	return VINF_SUCCESS;
522	}
523
524
525	/**
526	* Reads the VM-exit instruction length field from the VMCS into the VMX
527	* transient structure.
528	*
529	* @returns VBox status code.
530	* @param pVCpu Pointer to the VMCPU.
531	* @param pVmxTransient Pointer to the VMX transient structure.
532	*/
533	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
534	{
535	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
536	{
537	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
538	AssertRCReturn(rc, rc);
539	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
540	}
541	return VINF_SUCCESS;
542	}
543
544
545	/**
546	* Reads the exit qualification from the VMCS into the VMX transient structure.
547	*
548	* @returns VBox status code.
549	* @param pVCpu Pointer to the VMCPU.
550	* @param pVmxTransient Pointer to the VMX transient structure.
551	*/
552	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
553	{
554	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
555	{
556	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
557	AssertRCReturn(rc, rc);
558	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
559	}
560	return VINF_SUCCESS;
561	}
562
563
564	/**
565	* Reads the IDT-vectoring information field from the VMCS into the VMX
566	* transient structure.
567	*
568	* @returns VBox status code.
569	* @param pVmxTransient Pointer to the VMX transient structure.
570	*
571	* @remarks No-long-jump zone!!!
572	*/
573	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
574	{
575	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
576	{
577	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
578	AssertRCReturn(rc, rc);
579	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
580	}
581	return VINF_SUCCESS;
582	}
583
584
585	/**
586	* Reads the IDT-vectoring error code from the VMCS into the VMX
587	* transient structure.
588	*
589	* @returns VBox status code.
590	* @param pVmxTransient Pointer to the VMX transient structure.
591	*/
592	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
593	{
594	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
595	{
596	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
597	AssertRCReturn(rc, rc);
598	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
599	}
600	return VINF_SUCCESS;
601	}
602
603
604	/**
605	* Enters VMX root mode operation on the current CPU.
606	*
607	* @returns VBox status code.
608	* @param pVM Pointer to the VM (optional, can be NULL, after
609	* a resume).
610	* @param HCPhysCpuPage Physical address of the VMXON region.
611	* @param pvCpuPage Pointer to the VMXON region.
612	*/
613	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
614	{
615	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
616	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
617
618	if (pVM)
619	{
620	/* Write the VMCS revision dword to the VMXON region. */
621	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
622	}
623
624	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
625	RTCCUINTREG fFlags = ASMIntDisableFlags();
626
627	/* Enable the VMX bit in CR4 if necessary. */
628	RTCCUINTREG uCr4 = ASMGetCR4();
629	if (!(uCr4 & X86_CR4_VMXE))
630	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
631
632	/* Enter VMX root mode. */
633	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
634	if (RT_FAILURE(rc))
635	ASMSetCR4(uCr4);
636
637	/* Restore interrupts. */
638	ASMSetFlags(fFlags);
639	return rc;
640	}
641
642
643	/**
644	* Exits VMX root mode operation on the current CPU.
645	*
646	* @returns VBox status code.
647	*/
648	static int hmR0VmxLeaveRootMode(void)
649	{
650	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
651	RTCCUINTREG fFlags = ASMIntDisableFlags();
652	int rc = VINF_SUCCESS;
653
654	/* If we're for some reason not in VMX root mode, then don't leave it. */
655	if (ASMGetCR4() & X86_CR4_VMXE)
656	{
657	/* Exit VMX root mode and clear the VMX bit in CR4 */
658	VMXDisable();
659	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
660	}
661	else
662	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
663
664	/* Restore interrupts. */
665	ASMSetFlags(fFlags);
666	return rc;
667	}
668
669
670	/**
671	* Allocates and maps one physically contiguous page. The allocated page is
672	* zero'd out. (Used by various VT-x structures).
673	*
674	* @returns IPRT status code.
675	* @param pMemObj Pointer to the ring-0 memory object.
676	* @param ppVirt Where to store the virtual address of the
677	* allocation.
678	* @param pPhys Where to store the physical address of the
679	* allocation.
680	*/
681	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
682	{
683	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
684	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
685	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
686
687	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
688	if (RT_FAILURE(rc))
689	return rc;
690	ppVirt = RTR0MemObjAddress(pMemObj);
691	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
692	ASMMemZero32(*ppVirt, PAGE_SIZE);
693	return VINF_SUCCESS;
694	}
695
696
697	/**
698	* Frees and unmaps an allocated physical page.
699	*
700	* @param pMemObj Pointer to the ring-0 memory object.
701	* @param ppVirt Where to re-initialize the virtual address of
702	* allocation as 0.
703	* @param pHCPhys Where to re-initialize the physical address of the
704	* allocation as 0.
705	*/
706	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
707	{
708	AssertPtr(pMemObj);
709	AssertPtr(ppVirt);
710	AssertPtr(pHCPhys);
711	if (*pMemObj != NIL_RTR0MEMOBJ)
712	{
713	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
714	AssertRC(rc);
715	*pMemObj = NIL_RTR0MEMOBJ;
716	*ppVirt = 0;
717	*pHCPhys = 0;
718	}
719	}
720
721
722	/**
723	* Worker function to free VT-x related structures.
724	*
725	* @returns IPRT status code.
726	* @param pVM Pointer to the VM.
727	*/
728	static void hmR0VmxStructsFree(PVM pVM)
729	{
730	for (VMCPUID i = 0; i < pVM->cCpus; i++)
731	{
732	PVMCPU pVCpu = &pVM->aCpus[i];
733	AssertPtr(pVCpu);
734
735	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
736	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
737
738	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
739	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
740
741	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
742	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
743	}
744
745	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
746	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
747	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
748	#endif
749	}
750
751
752	/**
753	* Worker function to allocate VT-x related VM structures.
754	*
755	* @returns IPRT status code.
756	* @param pVM Pointer to the VM.
757	*/
758	static int hmR0VmxStructsAlloc(PVM pVM)
759	{
760	/*
761	* Initialize members up-front so we can cleanup properly on allocation failure.
762	*/
763	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
764	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
765	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
766	pVM->hm.s.vmx.HCPhys##a_Name = 0;
767
768	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
769	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
770	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
771	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
772
773	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
774	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
775	#endif
776	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
777
778	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
779	for (VMCPUID i = 0; i < pVM->cCpus; i++)
780	{
781	PVMCPU pVCpu = &pVM->aCpus[i];
782	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
783	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
784	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
785	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
786	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
787	}
788	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
789	#undef VMXLOCAL_INIT_VM_MEMOBJ
790
791	/*
792	* Allocate all the VT-x structures.
793	*/
794	int rc = VINF_SUCCESS;
795	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
796	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
797	if (RT_FAILURE(rc))
798	goto cleanup;
799	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
800	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
801	#endif
802
803	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
804	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
805	{
806	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
807	&pVM->hm.s.vmx.HCPhysApicAccess);
808	if (RT_FAILURE(rc))
809	goto cleanup;
810	}
811
812	/*
813	* Initialize per-VCPU VT-x structures.
814	*/
815	for (VMCPUID i =0; i < pVM->cCpus; i++)
816	{
817	PVMCPU pVCpu = &pVM->aCpus[i];
818	AssertPtr(pVCpu);
819
820	/* Allocate the VM control structure (VMCS). */
821	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
822	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825
826	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
827	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
828	{
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
830	&pVCpu->hm.s.vmx.HCPhysVirtApic);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833	}
834
835	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
836	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
837	{
838	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
839	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
840	if (RT_FAILURE(rc))
841	goto cleanup;
842	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
843	}
844
845	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
846	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
847	if (RT_FAILURE(rc))
848	goto cleanup;
849
850	/* Allocate the VM-exit MSR-load page for the host MSRs. */
851	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
852	if (RT_FAILURE(rc))
853	goto cleanup;
854	}
855
856	return VINF_SUCCESS;
857
858	cleanup:
859	hmR0VmxStructsFree(pVM);
860	return rc;
861	}
862
863
864	/**
865	* Does global VT-x initialization (called during module initialization).
866	*
867	* @returns VBox status code.
868	*/
869	VMMR0DECL(int) VMXR0GlobalInit(void)
870	{
871	#ifdef HMVMX_USE_FUNCTION_TABLE
872	/* Setup the main VM exit handlers. */
873	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
874	# ifdef VBOX_STRICT
875	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
876	Assert(g_apfnVMExitHandlers[i]);
877	# endif
878	#endif
879	return VINF_SUCCESS;
880	}
881
882
883	/**
884	* Does global VT-x termination (called during module termination).
885	*/
886	VMMR0DECL(void) VMXR0GlobalTerm()
887	{
888	/* Nothing to do currently. */
889	}
890
891
892	/**
893	* Sets up and activates VT-x on the current CPU.
894	*
895	* @returns VBox status code.
896	* @param pCpu Pointer to the global CPU info struct.
897	* @param pVM Pointer to the VM (can be NULL after a host resume
898	* operation).
899	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
900	* fEnabledByHost is true).
901	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
902	* @a fEnabledByHost is true).
903	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
904	* enable VT-x/AMD-V on the host.
905	*/
906	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
907	{
908	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
909	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
910
911	if (!fEnabledByHost)
912	{
913	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
914	if (RT_FAILURE(rc))
915	return rc;
916	}
917
918	/*
919	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
920	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
921	* each time while reusing a VPID after hitting the MaxASID limit once.
922	*/
923	if ( pVM
924	&& pVM->hm.s.vmx.fVpid
925	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
926	{
927	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
928	pCpu->fFlushAsidBeforeUse = false;
929	}
930	else
931	pCpu->fFlushAsidBeforeUse = true;
932
933	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
934	++pCpu->cTlbFlushes;
935
936	return VINF_SUCCESS;
937	}
938
939
940	/**
941	* Deactivates VT-x on the current CPU.
942	*
943	* @returns VBox status code.
944	* @param pCpu Pointer to the global CPU info struct.
945	* @param pvCpuPage Pointer to the VMXON region.
946	* @param HCPhysCpuPage Physical address of the VMXON region.
947	*/
948	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
949	{
950	NOREF(pCpu);
951	NOREF(pvCpuPage);
952	NOREF(HCPhysCpuPage);
953
954	hmR0VmxLeaveRootMode();
955	return VINF_SUCCESS;
956	}
957
958
959	/**
960	* Sets the permission bits for the specified MSR in the MSR bitmap.
961	*
962	* @param pVCpu Pointer to the VMCPU.
963	* @param uMSR The MSR value.
964	* @param enmRead Whether reading this MSR causes a VM-exit.
965	* @param enmWrite Whether writing this MSR causes a VM-exit.
966	*/
967	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
968	{
969	int32_t iBit;
970	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
971
972	/*
973	* Layout:
974	* 0x000 - 0x3ff - Low MSR read bits
975	* 0x400 - 0x7ff - High MSR read bits
976	* 0x800 - 0xbff - Low MSR write bits
977	* 0xc00 - 0xfff - High MSR write bits
978	*/
979	if (uMsr <= 0x00001FFF)
980	{
981	/* Pentium-compatible MSRs */
982	iBit = uMsr;
983	}
984	else if ( uMsr >= 0xC0000000
985	&& uMsr <= 0xC0001FFF)
986	{
987	/* AMD Sixth Generation x86 Processor MSRs */
988	iBit = (uMsr - 0xC0000000);
989	pbMsrBitmap += 0x400;
990	}
991	else
992	{
993	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
994	return;
995	}
996
997	Assert(iBit <= 0x1fff);
998	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
999	ASMBitSet(pbMsrBitmap, iBit);
1000	else
1001	ASMBitClear(pbMsrBitmap, iBit);
1002
1003	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1004	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1005	else
1006	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1007	}
1008
1009
1010	/**
1011	* Flushes the TLB using EPT.
1012	*
1013	* @returns VBox status code.
1014	* @param pVM Pointer to the VM.
1015	* @param pVCpu Pointer to the VMCPU.
1016	* @param enmFlush Type of flush.
1017	*/
1018	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1019	{
1020	AssertPtr(pVM);
1021	Assert(pVM->hm.s.fNestedPaging);
1022
1023	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1024
1025	uint64_t descriptor[2];
1026	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1027	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1028
1029	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1030	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1031	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1032	}
1033
1034
1035	/**
1036	* Flushes the TLB using VPID.
1037	*
1038	* @returns VBox status code.
1039	* @param pVM Pointer to the VM.
1040	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1041	* enmFlush).
1042	* @param enmFlush Type of flush.
1043	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1044	* on @a enmFlush).
1045	*/
1046	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1047	{
1048	AssertPtr(pVM);
1049	Assert(pVM->hm.s.vmx.fVpid);
1050
1051	uint64_t descriptor[2];
1052	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1053	{
1054	descriptor[0] = 0;
1055	descriptor[1] = 0;
1056	}
1057	else
1058	{
1059	AssertPtr(pVCpu);
1060	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1061	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1062	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1063	descriptor[1] = GCPtr;
1064	}
1065
1066	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1067	AssertMsg(rc == VINF_SUCCESS,
1068	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1069	if ( RT_SUCCESS(rc)
1070	&& pVCpu)
1071	{
1072	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1073	}
1074	}
1075
1076
1077	/**
1078	* Invalidates a guest page by guest virtual address. Only relevant for
1079	* EPT/VPID, otherwise there is nothing really to invalidate.
1080	*
1081	* @returns VBox status code.
1082	* @param pVM Pointer to the VM.
1083	* @param pVCpu Pointer to the VMCPU.
1084	* @param GCVirt Guest virtual address of the page to invalidate.
1085	*/
1086	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1087	{
1088	AssertPtr(pVM);
1089	AssertPtr(pVCpu);
1090	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1091
1092	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1093	if (!fFlushPending)
1094	{
1095	/*
1096	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1097	* See @bugref{6043} and @bugref{6177}.
1098	*
1099	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1100	* function maybe called in a loop with individual addresses.
1101	*/
1102	if (pVM->hm.s.vmx.fVpid)
1103	{
1104	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1105	{
1106	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1107	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1108	}
1109	else
1110	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1111	}
1112	else if (pVM->hm.s.fNestedPaging)
1113	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	}
1115
1116	return VINF_SUCCESS;
1117	}
1118
1119
1120	/**
1121	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1122	* otherwise there is nothing really to invalidate.
1123	*
1124	* @returns VBox status code.
1125	* @param pVM Pointer to the VM.
1126	* @param pVCpu Pointer to the VMCPU.
1127	* @param GCPhys Guest physical address of the page to invalidate.
1128	*/
1129	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1130	{
1131	LogFlowFunc(("%RGp\n", GCPhys));
1132
1133	/*
1134	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1135	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1136	* This function might be called in a loop.
1137	*/
1138	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1139	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1140	return VINF_SUCCESS;
1141	}
1142
1143
1144	/**
1145	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1146	* case where neither EPT nor VPID is supported by the CPU.
1147	*
1148	* @param pVM Pointer to the VM.
1149	* @param pVCpu Pointer to the VMCPU.
1150	*
1151	* @remarks Called with interrupts disabled.
1152	*/
1153	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1154	{
1155	NOREF(pVM);
1156	AssertPtr(pVCpu);
1157	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1158	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1159
1160	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1161	AssertPtr(pCpu);
1162
1163	pVCpu->hm.s.TlbShootdown.cPages = 0;
1164	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1165	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1166	pVCpu->hm.s.fForceTLBFlush = false;
1167	return;
1168	}
1169
1170
1171	/**
1172	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1173	*
1174	* @param pVM Pointer to the VM.
1175	* @param pVCpu Pointer to the VMCPU.
1176	* @remarks All references to "ASID" in this function pertains to "VPID" in
1177	* Intel's nomenclature. The reason is, to avoid confusion in compare
1178	* statements since the host-CPU copies are named "ASID".
1179	*
1180	* @remarks Called with interrupts disabled.
1181	*/
1182	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1183	{
1184	AssertPtr(pVM);
1185	AssertPtr(pVCpu);
1186	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1187	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1188	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1189
1190	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1191	AssertPtr(pCpu);
1192
1193	/*
1194	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1195	* This can happen both for start & resume due to long jumps back to ring-3.
1196	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1197	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1198	*/
1199	bool fNewASID = false;
1200	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1201	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1202	{
1203	pVCpu->hm.s.fForceTLBFlush = true;
1204	fNewASID = true;
1205	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1206	}
1207
1208	/*
1209	* Check for explicit TLB shootdowns.
1210	*/
1211	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1212	{
1213	pVCpu->hm.s.fForceTLBFlush = true;
1214	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1215	}
1216
1217	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1218	if (pVCpu->hm.s.fForceTLBFlush)
1219	{
1220	if (fNewASID)
1221	{
1222	++pCpu->uCurrentAsid;
1223	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1224	{
1225	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1226	pCpu->cTlbFlushes++;
1227	pCpu->fFlushAsidBeforeUse = true;
1228	}
1229
1230	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1231	if (pCpu->fFlushAsidBeforeUse)
1232	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1233	}
1234	else
1235	{
1236	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1237	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1238	else
1239	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1240	}
1241
1242	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1243	pVCpu->hm.s.fForceTLBFlush = false;
1244	}
1245	else
1246	{
1247	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1248	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1249	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1250	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1251
1252	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1253	* not be executed. See hmQueueInvlPage() where it is commented
1254	* out. Support individual entry flushing someday. */
1255	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1256	{
1257	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1258
1259	/*
1260	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1261	* as supported by the CPU.
1262	*/
1263	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1264	{
1265	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1266	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1267	}
1268	else
1269	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1270	}
1271	else
1272	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1273	}
1274	pVCpu->hm.s.TlbShootdown.cPages = 0;
1275	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1276
1277	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1278	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1279	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1280	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1281	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1282	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1283
1284	/* Update VMCS with the VPID. */
1285	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1286	AssertRC(rc);
1287	}
1288
1289
1290	/**
1291	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1292	*
1293	* @returns VBox status code.
1294	* @param pVM Pointer to the VM.
1295	* @param pVCpu Pointer to the VMCPU.
1296	*
1297	* @remarks Called with interrupts disabled.
1298	*/
1299	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1300	{
1301	AssertPtr(pVM);
1302	AssertPtr(pVCpu);
1303	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1304	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1305
1306	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1307	AssertPtr(pCpu);
1308
1309	/*
1310	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1311	* This can happen both for start & resume due to long jumps back to ring-3.
1312	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1313	*/
1314	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1315	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1316	{
1317	pVCpu->hm.s.fForceTLBFlush = true;
1318	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1319	}
1320
1321	/* Check for explicit TLB shootdown flushes. */
1322	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1323	{
1324	pVCpu->hm.s.fForceTLBFlush = true;
1325	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1326	}
1327
1328	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1329	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1330
1331	if (pVCpu->hm.s.fForceTLBFlush)
1332	{
1333	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1334	pVCpu->hm.s.fForceTLBFlush = false;
1335	}
1336	else
1337	{
1338	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1339	* not be executed. See hmQueueInvlPage() where it is commented
1340	* out. Support individual entry flushing someday. */
1341	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1342	{
1343	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1344	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1345	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1346	}
1347	else
1348	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1349	}
1350
1351	pVCpu->hm.s.TlbShootdown.cPages = 0;
1352	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1353	}
1354
1355
1356	/**
1357	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1358	*
1359	* @returns VBox status code.
1360	* @param pVM Pointer to the VM.
1361	* @param pVCpu Pointer to the VMCPU.
1362	*
1363	* @remarks Called with interrupts disabled.
1364	*/
1365	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1366	{
1367	AssertPtr(pVM);
1368	AssertPtr(pVCpu);
1369	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1370	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1371
1372	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1373
1374	/*
1375	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1376	* This can happen both for start & resume due to long jumps back to ring-3.
1377	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1378	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1379	*/
1380	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1381	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1382	{
1383	pVCpu->hm.s.fForceTLBFlush = true;
1384	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1385	}
1386
1387	/* Check for explicit TLB shootdown flushes. */
1388	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1389	{
1390	/*
1391	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1392	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1393	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1394	*/
1395	pVCpu->hm.s.fForceTLBFlush = true;
1396	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1397	}
1398
1399	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1400	if (pVCpu->hm.s.fForceTLBFlush)
1401	{
1402	++pCpu->uCurrentAsid;
1403	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1404	{
1405	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1406	pCpu->fFlushAsidBeforeUse = true;
1407	pCpu->cTlbFlushes++;
1408	}
1409
1410	pVCpu->hm.s.fForceTLBFlush = false;
1411	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1412	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1413	if (pCpu->fFlushAsidBeforeUse)
1414	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1415	}
1416	else
1417	{
1418	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1419	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1420	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1421	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1422
1423	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1424	* not be executed. See hmQueueInvlPage() where it is commented
1425	* out. Support individual entry flushing someday. */
1426	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1427	{
1428	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1429	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1430	{
1431	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1432	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1433	}
1434	else
1435	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1436	}
1437	else
1438	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1439	}
1440
1441	pVCpu->hm.s.TlbShootdown.cPages = 0;
1442	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1443
1444	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1445	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1446	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1447	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1448	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1449	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1450
1451	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1452	AssertRC(rc);
1453	}
1454
1455
1456	/**
1457	* Flushes the guest TLB entry based on CPU capabilities.
1458	*
1459	* @param pVCpu Pointer to the VMCPU.
1460	*/
1461	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1462	{
1463	PVM pVM = pVCpu->CTX_SUFF(pVM);
1464	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1465	{
1466	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1467	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1468	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1469	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1470	default:
1471	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1472	break;
1473	}
1474	}
1475
1476
1477	/**
1478	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1479	* TLB entries from the host TLB before VM-entry.
1480	*
1481	* @returns VBox status code.
1482	* @param pVM Pointer to the VM.
1483	*/
1484	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1485	{
1486	/*
1487	* Determine optimal flush type for nested paging.
1488	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1489	* guest execution (see hmR3InitFinalizeR0()).
1490	*/
1491	if (pVM->hm.s.fNestedPaging)
1492	{
1493	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1494	{
1495	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1496	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1497	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1498	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1499	else
1500	{
1501	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1502	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1503	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1504	}
1505
1506	/* Make sure the write-back cacheable memory type for EPT is supported. */
1507	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1508	{
1509	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1510	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1511	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1512	}
1513	}
1514	else
1515	{
1516	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1518	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1519	}
1520	}
1521
1522	/*
1523	* Determine optimal flush type for VPID.
1524	*/
1525	if (pVM->hm.s.vmx.fVpid)
1526	{
1527	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1528	{
1529	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1530	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1531	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1532	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1533	else
1534	{
1535	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1536	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1537	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1538	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1539	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1540	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1541	pVM->hm.s.vmx.fVpid = false;
1542	}
1543	}
1544	else
1545	{
1546	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1547	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1548	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1549	pVM->hm.s.vmx.fVpid = false;
1550	}
1551	}
1552
1553	/*
1554	* Setup the handler for flushing tagged-TLBs.
1555	*/
1556	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1557	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1558	else if (pVM->hm.s.fNestedPaging)
1559	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1560	else if (pVM->hm.s.vmx.fVpid)
1561	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1562	else
1563	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1564	return VINF_SUCCESS;
1565	}
1566
1567
1568	/**
1569	* Sets up pin-based VM-execution controls in the VMCS.
1570	*
1571	* @returns VBox status code.
1572	* @param pVM Pointer to the VM.
1573	* @param pVCpu Pointer to the VMCPU.
1574	*/
1575	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1576	{
1577	AssertPtr(pVM);
1578	AssertPtr(pVCpu);
1579
1580	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1581	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1582
1583	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1584	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1585	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1586
1587	/* Enable the VMX preemption timer. */
1588	if (pVM->hm.s.vmx.fUsePreemptTimer)
1589	{
1590	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1591	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1592	}
1593
1594	if ((val & zap) != val)
1595	{
1596	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1597	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1598	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1599	}
1600
1601	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1602	AssertRCReturn(rc, rc);
1603
1604	/* Update VCPU with the currently set pin-based VM-execution controls. */
1605	pVCpu->hm.s.vmx.u32PinCtls = val;
1606	return rc;
1607	}
1608
1609
1610	/**
1611	* Sets up processor-based VM-execution controls in the VMCS.
1612	*
1613	* @returns VBox status code.
1614	* @param pVM Pointer to the VM.
1615	* @param pVMCPU Pointer to the VMCPU.
1616	*/
1617	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1618	{
1619	AssertPtr(pVM);
1620	AssertPtr(pVCpu);
1621
1622	int rc = VERR_INTERNAL_ERROR_5;
1623	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1624	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1625
1626	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1627	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1628	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1629	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1630	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1632	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1633
1634	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1635	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1636	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1637	{
1638	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1639	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1640	}
1641
1642	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1643	if (!pVM->hm.s.fNestedPaging)
1644	{
1645	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1646	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1647	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1648	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1649	}
1650
1651	/* Use TPR shadowing if supported by the CPU. */
1652	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1653	{
1654	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1655	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1656	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1657	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1658	AssertRCReturn(rc, rc);
1659
1660	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1661	/* CR8 writes causes a VM-exit based on TPR threshold. */
1662	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1663	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1664	}
1665	else
1666	{
1667	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1668	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1669	}
1670
1671	/* Use MSR-bitmaps if supported by the CPU. */
1672	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1673	{
1674	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1675
1676	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1677	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1678	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1679	AssertRCReturn(rc, rc);
1680
1681	/*
1682	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1683	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1684	*/
1685	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1686	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1687	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1688	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1689	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1690	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1691	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1692	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1693	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1694	}
1695
1696	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1697	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1698	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1699
1700	if ((val & zap) != val)
1701	{
1702	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1703	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1704	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1705	}
1706
1707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1708	AssertRCReturn(rc, rc);
1709
1710	/* Update VCPU with the currently set processor-based VM-execution controls. */
1711	pVCpu->hm.s.vmx.u32ProcCtls = val;
1712
1713	/*
1714	* Secondary processor-based VM-execution controls.
1715	*/
1716	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1717	{
1718	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1719	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1720
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1722	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1723
1724	if (pVM->hm.s.fNestedPaging)
1725	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1726	else
1727	{
1728	/*
1729	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1730	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1731	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1732	*/
1733	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1734	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1735	}
1736
1737	if (pVM->hm.s.vmx.fVpid)
1738	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1739
1740	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1741	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1742
1743	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1744	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1745	* done dynamically. */
1746	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1747	{
1748	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1749	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1750	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1751	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1752	AssertRCReturn(rc, rc);
1753	}
1754
1755	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1756	{
1757	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1758	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1759	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1760	}
1761
1762	if ((val & zap) != val)
1763	{
1764	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1765	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1766	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1767	}
1768
1769	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1770	AssertRCReturn(rc, rc);
1771
1772	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1773	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1774	}
1775
1776	return VINF_SUCCESS;
1777	}
1778
1779
1780	/**
1781	* Sets up miscellaneous (everything other than Pin & Processor-based
1782	* VM-execution) control fields in the VMCS.
1783	*
1784	* @returns VBox status code.
1785	* @param pVM Pointer to the VM.
1786	* @param pVCpu Pointer to the VMCPU.
1787	*/
1788	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1789	{
1790	AssertPtr(pVM);
1791	AssertPtr(pVCpu);
1792
1793	int rc = VERR_GENERAL_FAILURE;
1794
1795	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1796	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1797
1798	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1799
1800	/*
1801	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1802	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1803	* We thus use the exception bitmap to control it rather than use both.
1804	*/
1805	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1806	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1807
1808	/** @todo Explore possibility of using IO-bitmaps. */
1809	/* All IO & IOIO instructions cause VM-exits. */
1810	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1811	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1812
1813	/* Setup MSR autoloading/autostoring. */
1814	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1815	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1816	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1817	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1818	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1819	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1820
1821	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1822	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1823	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1824	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1825
1826	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1827	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1828
1829	/* Setup debug controls */
1830	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1831	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1832	AssertRCReturn(rc, rc);
1833	return rc;
1834	}
1835
1836
1837	/**
1838	* Sets up the initial exception bitmap in the VMCS based on static conditions
1839	* (i.e. conditions that cannot ever change at runtime).
1840	*
1841	* @returns VBox status code.
1842	* @param pVM Pointer to the VM.
1843	* @param pVCpu Pointer to the VMCPU.
1844	*/
1845	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1846	{
1847	AssertPtr(pVM);
1848	AssertPtr(pVCpu);
1849
1850	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1851
1852	uint32_t u32XcptBitmap = 0;
1853
1854	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1855	if (!pVM->hm.s.fNestedPaging)
1856	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1857
1858	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1859	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1860	AssertRCReturn(rc, rc);
1861	return rc;
1862	}
1863
1864
1865	/**
1866	* Sets up the initial guest-state mask. The guest-state mask is consulted
1867	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1868	* for the nested virtualization case (as it would cause a VM-exit).
1869	*
1870	* @param pVCpu Pointer to the VMCPU.
1871	*/
1872	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1873	{
1874	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1875	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1876	return VINF_SUCCESS;
1877	}
1878
1879
1880	/**
1881	* Does per-VM VT-x initialization.
1882	*
1883	* @returns VBox status code.
1884	* @param pVM Pointer to the VM.
1885	*/
1886	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1887	{
1888	LogFlowFunc(("pVM=%p\n", pVM));
1889
1890	int rc = hmR0VmxStructsAlloc(pVM);
1891	if (RT_FAILURE(rc))
1892	{
1893	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1894	return rc;
1895	}
1896
1897	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1898	{
1899	PVMCPU pVCpu = &pVM->aCpus[i];
1900
1901	/* Current guest paging mode. */
1902	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1903	}
1904
1905	return VINF_SUCCESS;
1906	}
1907
1908
1909	/**
1910	* Does per-VM VT-x termination.
1911	*
1912	* @returns VBox status code.
1913	* @param pVM Pointer to the VM.
1914	*/
1915	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1916	{
1917	LogFlowFunc(("pVM=%p\n", pVM));
1918
1919	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1920	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1921	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1922	#endif
1923	hmR0VmxStructsFree(pVM);
1924	return VINF_SUCCESS;
1925	}
1926
1927
1928	/**
1929	* Sets up the VM for execution under VT-x.
1930	* This function is only called once per-VM during initalization.
1931	*
1932	* @returns VBox status code.
1933	* @param pVM Pointer to the VM.
1934	*/
1935	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1936	{
1937	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1938	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1939
1940	LogFlowFunc(("pVM=%p\n", pVM));
1941
1942	/*
1943	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1944	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1945	*/
1946	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1947	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1948	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1949	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1950	{
1951	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1952	return VERR_INTERNAL_ERROR;
1953	}
1954
1955	/* Initialize these always, see hmR3InitFinalizeR0().*/
1956	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1957	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1958
1959	/* Setup the tagged-TLB flush handlers. */
1960	int rc = hmR0VmxSetupTaggedTlb(pVM);
1961	if (RT_FAILURE(rc))
1962	{
1963	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1964	return rc;
1965	}
1966
1967	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1968	{
1969	PVMCPU pVCpu = &pVM->aCpus[i];
1970	AssertPtr(pVCpu);
1971	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1972
1973	/* Set revision dword at the beginning of the VMCS structure. */
1974	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1975
1976	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1977	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1978	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1979	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1980
1981	/* Load this VMCS as the current VMCS. */
1982	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1983	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1984	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1985
1986	/* Setup the pin-based VM-execution controls. */
1987	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1988	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1989	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1990
1991	/* Setup the processor-based VM-execution controls. */
1992	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1993	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1994	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1995
1996	/* Setup the rest (miscellaneous) VM-execution controls. */
1997	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000
2001	/* Setup the initial exception bitmap. */
2002	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2003	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2004	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2005
2006	/* Setup the initial guest-state mask. */
2007	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2008	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2009	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2010
2011	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2012	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
2013	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2014	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2015	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2016	#endif
2017
2018	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2019	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2020	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2021	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2022
2023	/* Update the last error record for this VCPU. */
2024	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2025	}
2026
2027	return VINF_SUCCESS;
2028	}
2029
2030
2031	/**
2032	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2033	* the VMCS.
2034	*
2035	* @returns VBox status code.
2036	* @param pVM Pointer to the VM.
2037	* @param pVCpu Pointer to the VMCPU.
2038	*/
2039	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2040	{
2041	RTCCUINTREG uReg = ASMGetCR0();
2042	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2043
2044	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2045	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2046	if (HMVMX_IS_64BIT_HOST_MODE())
2047	{
2048	uint64_t uReg = hmR0Get64bitCR3();
2049	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2050	}
2051	else
2052	#endif
2053	{
2054	uReg = ASMGetCR3();
2055	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2056	}
2057
2058	uReg = ASMGetCR4();
2059	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2060	AssertRCReturn(rc, rc);
2061	return rc;
2062	}
2063
2064
2065	/**
2066	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2067	* the host-state area in the VMCS.
2068	*
2069	* @returns VBox status code.
2070	* @param pVM Pointer to the VM.
2071	* @param pVCpu Pointer to the VMCPU.
2072	*/
2073	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2074	{
2075	int rc = VERR_INTERNAL_ERROR_5;
2076	RTSEL uSelCS = 0;
2077	RTSEL uSelSS = 0;
2078	RTSEL uSelDS = 0;
2079	RTSEL uSelES = 0;
2080	RTSEL uSelFS = 0;
2081	RTSEL uSelGS = 0;
2082	RTSEL uSelTR = 0;
2083
2084	/*
2085	* Host Selector registers.
2086	*/
2087	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2088	if (HMVMX_IS_64BIT_HOST_MODE())
2089	{
2090	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2091	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2092	}
2093	else
2094	{
2095	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2096	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2097	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2098	}
2099	#else
2100	uSelCS = ASMGetCS();
2101	uSelSS = ASMGetSS();
2102	#endif
2103
2104	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2105	/** @todo Verify if we have any platform that actually run with DS or ES with
2106	* RPL != 0 in kernel space. */
2107	uSelDS = 0;
2108	uSelES = 0;
2109	uSelFS = 0;
2110	uSelGS = 0;
2111	uSelTR = ASMGetTR();
2112
2113	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2114	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2115	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2116	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2117	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2118	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2119	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2120	Assert(uSelCS != 0);
2121	Assert(uSelTR != 0);
2122
2123	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2124	#if 0
2125	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2126	Assert(uSelSS != 0);
2127	#endif
2128
2129	/* Write these host selector fields into the host-state area in the VMCS. */
2130	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2131	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2132	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2133	#if 0
2134	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2135	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2136	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2137	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2138	#endif
2139	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2140	AssertRCReturn(rc, rc);
2141
2142	/*
2143	* Host GDTR and IDTR.
2144	*/
2145	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2146	* be safe to -not- save and restore GDTR and IDTR in the assembly
2147	* code and just do it here and don't care if the limits are zapped on
2148	* VM-exit. */
2149	RTGDTR Gdtr;
2150	RT_ZERO(Gdtr);
2151	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2152	if (HMVMX_IS_64BIT_HOST_MODE())
2153	{
2154	X86XDTR64 Gtr64;
2155	X86XDTR64 Idtr64;
2156	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2157	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2158	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2159	Gdtr.cbGdt = Gdtr64.cb;
2160	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2161	}
2162	else
2163	#endif
2164	{
2165	RTIDTR Idtr;
2166	ASMGetGDTR(&Gdtr);
2167	ASMGetIDTR(&Idtr);
2168	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2169	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2170	}
2171	AssertRCReturn(rc, rc);
2172
2173	/*
2174	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2175	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2176	*/
2177	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2178	{
2179	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2180	return VERR_VMX_INVALID_HOST_STATE;
2181	}
2182
2183	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2184	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2185	if (HMVMX_IS_64BIT_HOST_MODE())
2186	{
2187	/* We need the 64-bit TR base for hybrid darwin. */
2188	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2189	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2190	}
2191	else
2192	#endif
2193	{
2194	uintptr_t uTRBase;
2195	#if HC_ARCH_BITS == 64
2196	uTRBase = X86DESC64_BASE(pDesc);
2197	#else
2198	uTRBase = X86DESC_BASE(pDesc);
2199	#endif
2200	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2201	}
2202	AssertRCReturn(rc, rc);
2203
2204	/*
2205	* Host FS base and GS base.
2206	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2207	* would take care of the bases. In 64-bit, the MSRs come into play.
2208	*/
2209	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2210	if (HMVMX_IS_64BIT_HOST_MODE())
2211	{
2212	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2213	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2214	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2215	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2216	AssertRCReturn(rc, rc);
2217	}
2218	#endif
2219	return rc;
2220	}
2221
2222
2223	/**
2224	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2225	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2226	* the host after every successful VM exit.
2227	*
2228	* @returns VBox status code.
2229	* @param pVM Pointer to the VM.
2230	* @param pVCpu Pointer to the VMCPU.
2231	*/
2232	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2233	{
2234	AssertPtr(pVCpu);
2235	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2236
2237	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2238	uint32_t cHostMsrs = 0;
2239	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2240
2241	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2242	{
2243	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2244	pHostMsr->u32Reserved = 0;
2245	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2246	if (CPUMIsGuestInLongMode(pVCpu))
2247	{
2248	/* Must match the EFER value in our 64 bits switcher. */
2249	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2250	}
2251	else
2252	#endif
2253	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2254	pHostMsr++; cHostMsrs++;
2255	}
2256
2257	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2258	if (HMVMX_IS_64BIT_HOST_MODE())
2259	{
2260	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2261	pHostMsr->u32Reserved = 0;
2262	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2263	pHostMsr++; cHostMsrs++;
2264	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2265	pHostMsr->u32Reserved = 0;
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2267	pHostMsr++; cHostMsrs++;
2268	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2269	pHostMsr->u32Reserved = 0;
2270	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2271	pHostMsr++; cHostMsrs++;
2272	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2273	#if 0
2274	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2275	pMsr->u32Reserved = 0;
2276	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2277	pHostMsr++; cHostMsrs++;
2278	#endif
2279	}
2280	#endif
2281
2282	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2283	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2284	{
2285	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2286	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2287	}
2288
2289	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2290
2291	/*
2292	* Host Sysenter MSRs.
2293	*/
2294	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2295	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2296	if (HMVMX_IS_64BIT_HOST_MODE())
2297	{
2298	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2299	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2300	}
2301	else
2302	{
2303	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2304	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2305	}
2306	#elif HC_ARCH_BITS == 32
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2308	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2309	#else
2310	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2311	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2312	#endif
2313	AssertRCReturn(rc, rc);
2314
2315	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2316	* hmR0VmxSetupExitCtls() !! */
2317	return rc;
2318	}
2319
2320
2321	/**
2322	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2323	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2324	* controls".
2325	*
2326	* @returns VBox status code.
2327	* @param pVCpu Pointer to the VMCPU.
2328	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2329	* out-of-sync. Make sure to update the required fields
2330	* before using them.
2331	*
2332	* @remarks No-long-jump zone!!!
2333	*/
2334	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2335	{
2336	int rc = VINF_SUCCESS;
2337	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2338	{
2339	PVM pVM = pVCpu->CTX_SUFF(pVM);
2340	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2341	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2342
2343	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2344	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2345
2346	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2347	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2348	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2349	else
2350	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2351
2352	/*
2353	* The following should not be set (since we're not in SMM mode):
2354	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2355	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2356	*/
2357
2358	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2359	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2360	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2361
2362	if ((val & zap) != val)
2363	{
2364	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2365	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2366	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2367	}
2368
2369	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2370	AssertRCReturn(rc, rc);
2371
2372	/* Update VCPU with the currently set VM-exit controls. */
2373	pVCpu->hm.s.vmx.u32EntryCtls = val;
2374	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2375	}
2376	return rc;
2377	}
2378
2379
2380	/**
2381	* Sets up the VM-exit controls in the VMCS.
2382	*
2383	* @returns VBox status code.
2384	* @param pVM Pointer to the VM.
2385	* @param pVCpu Pointer to the VMCPU.
2386	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2387	* out-of-sync. Make sure to update the required fields
2388	* before using them.
2389	*
2390	* @remarks requires EFER.
2391	*/
2392	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2393	{
2394	int rc = VINF_SUCCESS;
2395	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2396	{
2397	PVM pVM = pVCpu->CTX_SUFF(pVM);
2398	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2399	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2400
2401	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2402	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2403
2404	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2405	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2406	if (HMVMX_IS_64BIT_HOST_MODE())
2407	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2408	else
2409	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2410	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2411	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2412	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2413	else
2414	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2415	#endif
2416
2417	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2418	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2419
2420	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2421	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2422	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2423	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2424	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2425
2426	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2427	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2428
2429	if ((val & zap) != val)
2430	{
2431	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2432	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2433	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2434	}
2435
2436	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2437	AssertRCReturn(rc, rc);
2438
2439	/* Update VCPU with the currently set VM-exit controls. */
2440	pVCpu->hm.s.vmx.u32ExitCtls = val;
2441	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2442	}
2443	return rc;
2444	}
2445
2446
2447	/**
2448	* Loads the guest APIC and related state.
2449	*
2450	* @returns VBox status code.
2451	* @param pVM Pointer to the VM.
2452	* @param pVCpu Pointer to the VMCPU.
2453	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2454	* out-of-sync. Make sure to update the required fields
2455	* before using them.
2456	*/
2457	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2458	{
2459	int rc = VINF_SUCCESS;
2460	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2461	{
2462	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2463	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2464	{
2465	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2466
2467	bool fPendingIntr = false;
2468	uint8_t u8GuestTpr = 0;
2469	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2470	AssertRCReturn(rc, rc);
2471
2472	/*
2473	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2474	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2475	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2476	* the interrupt when we VM-exit for other reasons.
2477	*/
2478	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2479	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2480	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2481	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2482
2483	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2484	AssertRCReturn(rc, rc);
2485
2486	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2487	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2488	{
2489	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2490	pMixedCtx->msrLSTAR = u8GuestTpr;
2491	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2492	{
2493	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2494	if (fPendingIntr)
2495	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2496	else
2497	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2498	}
2499	}
2500	}
2501
2502	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2503	}
2504	return rc;
2505	}
2506
2507
2508	/**
2509	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2510	*
2511	* @returns
2512	* @param pVCpu Pointer to the VMCPU.
2513	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2514	* out-of-sync. Make sure to update the required fields
2515	* before using them.
2516	*
2517	* @remarks No-long-jump zone!!!
2518	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2519	*/
2520	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2521	{
2522	/*
2523	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2524	* inhibit interrupts or clear any existing interrupt-inhibition.
2525	*/
2526	uint32_t uIntrState = 0;
2527	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2528	{
2529	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2530	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2531	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2532	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2533	{
2534	/*
2535	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2536	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2537	*/
2538	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2539	}
2540	else if (pMixedCtx->eflags.Bits.u1IF)
2541	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2542	else
2543	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2544	}
2545	return uIntrState;
2546	}
2547
2548
2549	/**
2550	* Loads the guest's interruptibility-state into the guest-state area in the
2551	* VMCS.
2552	*
2553	* @returns VBox status code.
2554	* @param pVCpu Pointer to the VMCPU.
2555	* @param uIntrState The interruptibility-state to set.
2556	*/
2557	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2558	{
2559	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2560	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2561	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2562	AssertRCReturn(rc, rc);
2563	return rc;
2564	}
2565
2566
2567	/**
2568	* Loads the guest's RIP into the guest-state area in the VMCS.
2569	*
2570	* @returns VBox status code.
2571	* @param pVCpu Pointer to the VMCPU.
2572	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2573	* out-of-sync. Make sure to update the required fields
2574	* before using them.
2575	*
2576	* @remarks No-long-jump zone!!!
2577	*/
2578	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2579	{
2580	int rc = VINF_SUCCESS;
2581	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2582	{
2583	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2584	AssertRCReturn(rc, rc);
2585	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2586	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2587	}
2588	return rc;
2589	}
2590
2591
2592	/**
2593	* Loads the guest's RSP into the guest-state area in the VMCS.
2594	*
2595	* @returns VBox status code.
2596	* @param pVCpu Pointer to the VMCPU.
2597	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2598	* out-of-sync. Make sure to update the required fields
2599	* before using them.
2600	*
2601	* @remarks No-long-jump zone!!!
2602	*/
2603	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2604	{
2605	int rc = VINF_SUCCESS;
2606	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2607	{
2608	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2609	AssertRCReturn(rc, rc);
2610	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2611	}
2612	return rc;
2613	}
2614
2615
2616	/**
2617	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2618	*
2619	* @returns VBox status code.
2620	* @param pVCpu Pointer to the VMCPU.
2621	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2622	* out-of-sync. Make sure to update the required fields
2623	* before using them.
2624	*
2625	* @remarks No-long-jump zone!!!
2626	*/
2627	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2628	{
2629	int rc = VINF_SUCCESS;
2630	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2631	{
2632	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2633	Let us assert it as such and use native-width VMWRITE. */
2634	X86RFLAGS uRFlags = pMixedCtx->rflags;
2635	Assert(uRFlags.u64 >> 32 == 0);
2636	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2637	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2638
2639	/*
2640	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2641	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2642	*/
2643	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2644	{
2645	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2646	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2647	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2648	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2649	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2650	}
2651
2652	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2653	AssertRCReturn(rc, rc);
2654
2655	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2656	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2657	}
2658	return rc;
2659	}
2660
2661
2662	/**
2663	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2664	*
2665	* @returns VBox status code.
2666	* @param pVCpu Pointer to the VMCPU.
2667	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2668	* out-of-sync. Make sure to update the required fields
2669	* before using them.
2670	*
2671	* @remarks No-long-jump zone!!!
2672	*/
2673	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2674	{
2675	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2676	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2677	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2678	return rc;
2679	}
2680
2681
2682	/**
2683	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2684	* in the VMCS.
2685	*
2686	* @returns VBox status code.
2687	* @param pVM Pointer to the VM.
2688	* @param pVCpu Pointer to the VMCPU.
2689	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2690	* out-of-sync. Make sure to update the required fields
2691	* before using them.
2692	*
2693	* @remarks No-long-jump zone!!!
2694	*/
2695	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2696	{
2697	int rc = VINF_SUCCESS;
2698	PVM pVM = pVCpu->CTX_SUFF(pVM);
2699
2700	/*
2701	* Guest CR0.
2702	* Guest FPU.
2703	*/
2704	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2705	{
2706	uint64_t u64GuestCR0 = pCtx->cr0;
2707
2708	/* The guest's view (read access) of its CR0 is unblemished. */
2709	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2710	AssertRCReturn(rc, rc);
2711	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2712
2713	/* Setup VT-x's view of the guest CR0. */
2714	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2715	if (pVM->hm.s.fNestedPaging)
2716	{
2717	if (CPUMIsGuestPagingEnabledEx(pCtx))
2718	{
2719	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2720	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2721	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2722	}
2723	else
2724	{
2725	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2726	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2727	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2728	}
2729
2730	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2731	AssertRCReturn(rc, rc);
2732	}
2733	else
2734	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2735
2736	/*
2737	* Guest FPU bits.
2738	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2739	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2740	*/
2741	u64GuestCR0 \|= X86_CR0_NE;
2742	bool fInterceptNM = false;
2743	if (CPUMIsGuestFPUStateActive(pVCpu))
2744	{
2745	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2746	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2747	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2748	}
2749	else
2750	{
2751	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2752	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2753	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2754	}
2755
2756	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2757	bool fInterceptMF = false;
2758	if (!(pCtx->cr0 & X86_CR0_NE))
2759	fInterceptMF = true;
2760
2761	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2762	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2763	{
2764	Assert(PDMVmmDevHeapIsEnabled(pVM));
2765	Assert(pVM->hm.s.vmx.pRealModeTSS);
2766	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2767	fInterceptNM = true;
2768	fInterceptMF = true;
2769	}
2770	else
2771	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2772
2773	if (fInterceptNM)
2774	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2775	else
2776	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2777
2778	if (fInterceptMF)
2779	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2780	else
2781	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2782
2783	/* Additional intercepts for debugging, define these yourself explicitly. */
2784	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2785	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2786	\| RT_BIT(X86_XCPT_DB)
2787	\| RT_BIT(X86_XCPT_DE)
2788	\| RT_BIT(X86_XCPT_NM)
2789	\| RT_BIT(X86_XCPT_UD)
2790	\| RT_BIT(X86_XCPT_NP)
2791	\| RT_BIT(X86_XCPT_SS)
2792	\| RT_BIT(X86_XCPT_GP)
2793	\| RT_BIT(X86_XCPT_PF)
2794	\| RT_BIT(X86_XCPT_MF);
2795	#elif defined(VBOX_ALWAYS_TRAP_PF)
2796	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2797	#endif
2798
2799	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2800
2801	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2802	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2803	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2804	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2805	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2806	else
2807	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2808
2809	u64GuestCR0 \|= uSetCR0;
2810	u64GuestCR0 &= uZapCR0;
2811	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2812
2813	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2814	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2815	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2816	Log(("Load: VMX_VMCS_GUEST_CR0=%RGv (uSetCR0=%#RGv uZapCR0=%#RGv)\n", u64GuestCR0, uSetCR0, uZapCR0));
2817
2818	/*
2819	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2820	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2821	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2822	*/
2823	uint64_t u64CR0Mask = 0;
2824	u64CR0Mask = X86_CR0_PE
2825	\| X86_CR0_NE
2826	\| X86_CR0_WP
2827	\| X86_CR0_PG
2828	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2829	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2830	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2831	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2832	u64CR0Mask &= ~X86_CR0_PE;
2833	/* Enable this later. */
2834	/* if (pVM->hm.s.fNestedPaging)
2835	u64CR0Mask &= ~X86_CR0_WP; */
2836
2837	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2838	if (fInterceptNM)
2839	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2840	else
2841	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2842
2843	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2844	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2845	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2846	AssertRCReturn(rc, rc);
2847
2848	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2849	}
2850
2851	/*
2852	* Guest CR2.
2853	* It's always loaded in the assembler code. Nothing to do here.
2854	*/
2855
2856	/*
2857	* Guest CR3.
2858	*/
2859	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2860	{
2861	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2862	if (pVM->hm.s.fNestedPaging)
2863	{
2864	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2865
2866	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2867	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2868	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2869	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2870
2871	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2872	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2873	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2874
2875	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2876	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2877	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2878	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2879
2880	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2881	AssertRCReturn(rc, rc);
2882	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2883
2884	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2885	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2886	{
2887	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2888	if (CPUMIsGuestInPAEModeEx(pCtx))
2889	{
2890	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2891	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2892	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2893	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2894	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2895	AssertRCReturn(rc, rc);
2896	}
2897
2898	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2899	have Unrestricted Execution to handle the guest when it's not using paging. */
2900	GCPhysGuestCR3 = pCtx->cr3;
2901	}
2902	else
2903	{
2904	/*
2905	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2906	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2907	* EPT takes care of translating it to host-physical addresses.
2908	*/
2909	RTGCPHYS GCPhys;
2910	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2911	Assert(PDMVmmDevHeapIsEnabled(pVM));
2912
2913	/* We obtain it here every time as the guest could have relocated this PCI region. */
2914	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2915	AssertRCReturn(rc, rc);
2916
2917	GCPhysGuestCR3 = GCPhys;
2918	}
2919	}
2920	else
2921	{
2922	/* Non-nested paging case, just use the hypervisor's CR3. */
2923	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2924	}
2925
2926	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2927	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2928	AssertRCReturn(rc, rc);
2929
2930	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2931	}
2932
2933	/*
2934	* Guest CR4.
2935	*/
2936	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2937	{
2938	uint32_t u64GuestCR4 = pCtx->cr4;
2939	Assert(!(pCtx->cr4 >> 32));
2940
2941	/* The guest's view of its CR4 is unblemished. */
2942	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2943	AssertRCReturn(rc, rc);
2944	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2945
2946	/* Setup VT-x's view of the guest CR4. */
2947	/*
2948	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2949	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2950	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2951	*/
2952	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2953	{
2954	Assert(pVM->hm.s.vmx.pRealModeTSS);
2955	Assert(PDMVmmDevHeapIsEnabled(pVM));
2956	u64GuestCR4 &= ~X86_CR4_VME;
2957	}
2958
2959	if (pVM->hm.s.fNestedPaging)
2960	{
2961	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2962	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2963	{
2964	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2965	u64GuestCR4 \|= X86_CR4_PSE;
2966	/* Our identity mapping is a 32 bits page directory. */
2967	u64GuestCR4 &= ~X86_CR4_PAE;
2968	}
2969	/* else use guest CR4.*/
2970	}
2971	else
2972	{
2973	/*
2974	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2975	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2976	*/
2977	switch (pVCpu->hm.s.enmShadowMode)
2978	{
2979	case PGMMODE_REAL: /* Real-mode. */
2980	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2981	case PGMMODE_32_BIT: /* 32-bit paging. */
2982	{
2983	u64GuestCR4 &= ~X86_CR4_PAE;
2984	break;
2985	}
2986
2987	case PGMMODE_PAE: /* PAE paging. */
2988	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2989	{
2990	u64GuestCR4 \|= X86_CR4_PAE;
2991	break;
2992	}
2993
2994	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2995	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2996	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2997	break;
2998	#endif
2999	default:
3000	AssertFailed();
3001	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3002	}
3003	}
3004
3005	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3006	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3007	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3008	u64GuestCR4 \|= uSetCR4;
3009	u64GuestCR4 &= uZapCR4;
3010
3011	/* Write VT-x's view of the guest CR4 into the VMCS. */
3012	Log(("Load: VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
3013	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
3014
3015	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3016	uint64_t u64CR4Mask = 0;
3017	u64CR4Mask = X86_CR4_VME
3018	\| X86_CR4_PAE
3019	\| X86_CR4_PGE
3020	\| X86_CR4_PSE
3021	\| X86_CR4_VMXE;
3022	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3023	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3024	AssertRCReturn(rc, rc);
3025
3026	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3027	}
3028	return rc;
3029	}
3030
3031
3032	/**
3033	* Loads the guest debug registers into the guest-state area in the VMCS.
3034	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3035	*
3036	* @returns VBox status code.
3037	* @param pVCpu Pointer to the VMCPU.
3038	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3039	* out-of-sync. Make sure to update the required fields
3040	* before using them.
3041	*
3042	* @remarks No-long-jump zone!!!
3043	*/
3044	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3045	{
3046	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3047	return VINF_SUCCESS;
3048
3049	#ifdef VBOX_STRICT
3050	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3051	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3052	{
3053	Assert((pMixedCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3054	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3055	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3056	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3057	}
3058	#endif
3059
3060	int rc = VERR_INTERNAL_ERROR_5;
3061	PVM pVM = pVCpu->CTX_SUFF(pVM);
3062	bool fInterceptDB = false;
3063	bool fInterceptMovDRx = false;
3064	if (DBGFIsStepping(pVCpu))
3065	{
3066	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3067	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3068	{
3069	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3070	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3071	AssertRCReturn(rc, rc);
3072	Assert(fInterceptDB == false);
3073	}
3074	else
3075	fInterceptDB = true;
3076	}
3077
3078	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3079	{
3080	if (!CPUMIsHyperDebugStateActive(pVCpu))
3081	{
3082	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3083	AssertRC(rc);
3084	}
3085	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3086	fInterceptMovDRx = true;
3087	}
3088	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3089	{
3090	if (!CPUMIsGuestDebugStateActive(pVCpu))
3091	{
3092	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3093	AssertRC(rc);
3094	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3095	}
3096	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3097	Assert(fInterceptMovDRx == false);
3098	}
3099	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3100	{
3101	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3102	fInterceptMovDRx = true;
3103	}
3104
3105	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3106	if (fInterceptDB)
3107	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3108	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3109	{
3110	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3111	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3112	#endif
3113	}
3114
3115	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3116	if (fInterceptMovDRx)
3117	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3118	else
3119	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3120
3121	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3122	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3123
3124	/* The guest's view of its DR7 is unblemished. */
3125	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3126
3127	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3128	return rc;
3129	}
3130
3131
3132	#ifdef VBOX_STRICT
3133	/**
3134	* Strict function to validate segment registers.
3135	*/
3136	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3137	{
3138	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3139	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3140	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3141	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3142	{
3143	/* Protected mode checks */
3144	/* CS */
3145	Assert(pCtx->cs.Attr.n.u1Present);
3146	Assert(!(pCtx->cs.Attr.u & 0xf00));
3147	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3148	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3149	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3150	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3151	\|\| (pCtx->cs.Attr.n.u1Granularity));
3152	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3153	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3154	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3155	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3156	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3157	else
3158	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3159	/* SS */
3160	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3161	{
3162	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3163	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3164	Assert(pCtx->ss.Attr.n.u1Present);
3165	Assert(!(pCtx->ss.Attr.u & 0xf00));
3166	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3167	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3168	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3169	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3170	\|\| (pCtx->ss.Attr.n.u1Granularity));
3171	}
3172	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3173	/* CR0 might not be up-to-date here always, hence disabled. */
3174	#if 0
3175	if (!pCtx->cr0 & X86_CR0_PE)
3176	Assert(!pCtx->ss.Attr.n.u2Dpl);
3177	#endif
3178	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3179	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3180	{
3181	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3182	Assert(pCtx->ds.Attr.n.u1Present);
3183	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3184	Assert(!(pCtx->ds.Attr.u & 0xf00));
3185	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3186	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3187	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3188	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3189	\|\| (pCtx->ds.Attr.n.u1Granularity));
3190	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3191	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3192	}
3193	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3194	{
3195	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3196	Assert(pCtx->es.Attr.n.u1Present);
3197	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3198	Assert(!(pCtx->es.Attr.u & 0xf00));
3199	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3200	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3201	\|\| !(pCtx->es.Attr.n.u1Granularity));
3202	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3203	\|\| (pCtx->es.Attr.n.u1Granularity));
3204	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3205	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3206	}
3207	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3208	{
3209	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3210	Assert(pCtx->fs.Attr.n.u1Present);
3211	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3212	Assert(!(pCtx->fs.Attr.u & 0xf00));
3213	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3214	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3215	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3216	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3217	\|\| (pCtx->fs.Attr.n.u1Granularity));
3218	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3219	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3220	}
3221	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3222	{
3223	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3224	Assert(pCtx->gs.Attr.n.u1Present);
3225	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3226	Assert(!(pCtx->gs.Attr.u & 0xf00));
3227	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3228	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3229	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3230	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3231	\|\| (pCtx->gs.Attr.n.u1Granularity));
3232	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3233	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3234	}
3235	/* 64-bit capable CPUs. */
3236	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3237	Assert(!(pCtx->cs.u64Base >> 32));
3238	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3239	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3240	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3241	# endif
3242	}
3243	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3244	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3245	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3246	{
3247	/* Real and v86 mode checks. */
3248	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3249	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3250	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3251	{
3252	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3253	}
3254	else
3255	{
3256	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3257	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3258	}
3259
3260	/* CS */
3261	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3262	Assert(pCtx->cs.u32Limit == 0xffff);
3263	Assert(u32CSAttr == 0xf3);
3264	/* SS */
3265	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3266	Assert(pCtx->ss.u32Limit == 0xffff);
3267	Assert(u32SSAttr == 0xf3);
3268	/* DS */
3269	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3270	Assert(pCtx->ds.u32Limit == 0xffff);
3271	Assert(u32DSAttr == 0xf3);
3272	/* ES */
3273	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3274	Assert(pCtx->es.u32Limit == 0xffff);
3275	Assert(u32ESAttr == 0xf3);
3276	/* FS */
3277	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3278	Assert(pCtx->fs.u32Limit == 0xffff);
3279	Assert(u32FSAttr == 0xf3);
3280	/* GS */
3281	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3282	Assert(pCtx->gs.u32Limit == 0xffff);
3283	Assert(u32GSAttr == 0xf3);
3284	/* 64-bit capable CPUs. */
3285	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3286	Assert(!(pCtx->cs.u64Base >> 32));
3287	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3288	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3289	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3290	# endif
3291	}
3292	}
3293	#endif /* VBOX_STRICT */
3294
3295
3296	/**
3297	* Writes a guest segment register into the guest-state area in the VMCS.
3298	*
3299	* @returns VBox status code.
3300	* @param pVCpu Pointer to the VMCPU.
3301	* @param idxSel Index of the selector in the VMCS.
3302	* @param idxLimit Index of the segment limit in the VMCS.
3303	* @param idxBase Index of the segment base in the VMCS.
3304	* @param idxAccess Index of the access rights of the segment in the VMCS.
3305	* @param pSelReg Pointer to the segment selector.
3306	* @param pCtx Pointer to the guest-CPU context.
3307	*
3308	* @remarks No-long-jump zone!!!
3309	*/
3310	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3311	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3312	{
3313	int rc;
3314	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3315	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3316	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3317	AssertRCReturn(rc, rc);
3318
3319	uint32_t u32Access = pSelReg->Attr.u;
3320	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3321	{
3322	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3323	u32Access = 0xf3;
3324	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3325	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3326	}
3327	else
3328	{
3329	/*
3330	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3331	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3332	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3333	* loaded in protected-mode have their attribute as 0.
3334	*/
3335	if (!u32Access)
3336	u32Access = HMVMX_SEL_UNUSABLE;
3337	}
3338
3339	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3340	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3341	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3342
3343	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3344	AssertRCReturn(rc, rc);
3345	return rc;
3346	}
3347
3348
3349	/**
3350	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3351	* into the guest-state area in the VMCS.
3352	*
3353	* @returns VBox status code.
3354	* @param pVM Pointer to the VM.
3355	* @param pVCPU Pointer to the VMCPU.
3356	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3357	* out-of-sync. Make sure to update the required fields
3358	* before using them.
3359	*
3360	* @remarks No-long-jump zone!!!
3361	*/
3362	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3363	{
3364	int rc = VERR_INTERNAL_ERROR_5;
3365	PVM pVM = pVCpu->CTX_SUFF(pVM);
3366
3367	/*
3368	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3369	*/
3370	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3371	{
3372	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3373	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3374	{
3375	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3376	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3377	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3378	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3379	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3380	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3381	}
3382
3383	#ifdef VBOX_WITH_REM
3384	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3385	{
3386	Assert(pVM->hm.s.vmx.pRealModeTSS);
3387	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3388	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3389	{
3390	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3391	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3392	&& enmGuestMode >= PGMMODE_PROTECTED)
3393	{
3394	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3395	in real-mode (e.g. OpenBSD 4.0) */
3396	REMFlushTBs(pVM);
3397	Log(("Load: Switch to protected mode detected!\n"));
3398	}
3399	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3400	}
3401	}
3402	#endif
3403	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3404	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3405	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3406	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3407	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3408	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3409	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3410	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3411	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3412	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3413	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3414	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3415	AssertRCReturn(rc, rc);
3416
3417	#ifdef VBOX_STRICT
3418	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3419	#endif
3420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3421	}
3422
3423	/*
3424	* Guest TR.
3425	*/
3426	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3427	{
3428	/*
3429	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3430	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3431	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3432	*/
3433	uint16_t u16Sel = 0;
3434	uint32_t u32Limit = 0;
3435	uint64_t u64Base = 0;
3436	uint32_t u32AccessRights = 0;
3437
3438	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3439	{
3440	u16Sel = pMixedCtx->tr.Sel;
3441	u32Limit = pMixedCtx->tr.u32Limit;
3442	u64Base = pMixedCtx->tr.u64Base;
3443	u32AccessRights = pMixedCtx->tr.Attr.u;
3444	}
3445	else
3446	{
3447	Assert(pVM->hm.s.vmx.pRealModeTSS);
3448	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3449
3450	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3451	RTGCPHYS GCPhys;
3452	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3453	AssertRCReturn(rc, rc);
3454
3455	X86DESCATTR DescAttr;
3456	DescAttr.u = 0;
3457	DescAttr.n.u1Present = 1;
3458	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3459
3460	u16Sel = 0;
3461	u32Limit = HM_VTX_TSS_SIZE;
3462	u64Base = GCPhys; /* in real-mode phys = virt. */
3463	u32AccessRights = DescAttr.u;
3464	}
3465
3466	/* Validate. */
3467	Assert(!(u16Sel & RT_BIT(2)));
3468	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3469	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3470	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3471	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3472	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3473	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3474	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3475	Assert( (u32Limit & 0xfff) == 0xfff
3476	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3477	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3478	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3479
3480	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3481	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3482	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3483	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3484	AssertRCReturn(rc, rc);
3485
3486	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3487	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3488	}
3489
3490	/*
3491	* Guest GDTR.
3492	*/
3493	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3494	{
3495	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3496	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3497	AssertRCReturn(rc, rc);
3498
3499	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3500	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3501	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3502	}
3503
3504	/*
3505	* Guest LDTR.
3506	*/
3507	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3508	{
3509	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3510	uint32_t u32Access = 0;
3511	if (!pMixedCtx->ldtr.Attr.u)
3512	u32Access = HMVMX_SEL_UNUSABLE;
3513	else
3514	u32Access = pMixedCtx->ldtr.Attr.u;
3515
3516	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3517	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3518	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3519	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3520	AssertRCReturn(rc, rc);
3521
3522	/* Validate. */
3523	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3524	{
3525	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3526	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3527	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3528	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3529	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3530	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3531	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3532	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3533	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3534	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3535	}
3536
3537	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3538	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3539	}
3540
3541	/*
3542	* Guest IDTR.
3543	*/
3544	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3545	{
3546	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3547	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3548	AssertRCReturn(rc, rc);
3549
3550	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3551	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3552	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3553	}
3554
3555	return VINF_SUCCESS;
3556	}
3557
3558
3559	/**
3560	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3561	* areas. These MSRs will automatically be loaded to the host CPU on every
3562	* successful VM entry and stored from the host CPU on every successful VM exit.
3563	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3564	*
3565	* @returns VBox status code.
3566	* @param pVCpu Pointer to the VMCPU.
3567	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3568	* out-of-sync. Make sure to update the required fields
3569	* before using them.
3570	*
3571	* @remarks No-long-jump zone!!!
3572	*/
3573	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3574	{
3575	AssertPtr(pVCpu);
3576	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3577
3578	/*
3579	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3580	*/
3581	int rc = VINF_SUCCESS;
3582	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3583	{
3584	PVM pVM = pVCpu->CTX_SUFF(pVM);
3585	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3586	uint32_t cGuestMsrs = 0;
3587
3588	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3589	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3590	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3591	if (fSupportsNX \|\| fSupportsLongMode)
3592	{
3593	/** @todo support save IA32_EFER, i.e.
3594	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3595	* guest EFER need not be part of the VM-entry MSR-load area. */
3596	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3597	pGuestMsr->u32Reserved = 0;
3598	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3599	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3600	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3601	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3602	pGuestMsr++; cGuestMsrs++;
3603	if (fSupportsLongMode)
3604	{
3605	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3606	pGuestMsr->u32Reserved = 0;
3607	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3608	pGuestMsr++; cGuestMsrs++;
3609	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3610	pGuestMsr->u32Reserved = 0;
3611	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3612	pGuestMsr++; cGuestMsrs++;
3613	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3614	pGuestMsr->u32Reserved = 0;
3615	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3616	pGuestMsr++; cGuestMsrs++;
3617	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3618	#if 0
3619	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3620	pGuestMsr->u32Reserved = 0;
3621	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3622	pGuestMsr++; cGuestMsrs++;
3623	#endif
3624	}
3625	}
3626
3627	/*
3628	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3629	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3630	*/
3631	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3632	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3633	{
3634	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3635	pGuestMsr->u32Reserved = 0;
3636	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3637	AssertRCReturn(rc, rc);
3638	pGuestMsr++; cGuestMsrs++;
3639	}
3640
3641	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3642	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3643	{
3644	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3645	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3646	}
3647
3648	/* Update the VCPU's copy of the guest MSR count. */
3649	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3650	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3651	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3652	AssertRCReturn(rc, rc);
3653
3654	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3655	}
3656
3657	/*
3658	* Guest Sysenter MSRs.
3659	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3660	* VM exits on WRMSRs for these MSRs.
3661	*/
3662	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3663	{
3664	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3665	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3666	}
3667	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3668	{
3669	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3670	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3671	}
3672	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3673	{
3674	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3675	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3676	}
3677	AssertRCReturn(rc, rc);
3678
3679	return rc;
3680	}
3681
3682
3683	/**
3684	* Loads the guest activity state into the guest-state area in the VMCS.
3685	*
3686	* @returns VBox status code.
3687	* @param pVCpu Pointer to the VMCPU.
3688	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3689	* out-of-sync. Make sure to update the required fields
3690	* before using them.
3691	*
3692	* @remarks No-long-jump zone!!!
3693	*/
3694	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3695	{
3696	/** @todo See if we can make use of other states, e.g.
3697	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3698	int rc = VINF_SUCCESS;
3699	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3700	{
3701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3702	AssertRCReturn(rc, rc);
3703	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3704	}
3705	return rc;
3706	}
3707
3708
3709	/**
3710	* Sets up the appropriate function to run guest code.
3711	*
3712	* @returns VBox status code.
3713	* @param pVCpu Pointer to the VMCPU.
3714	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3715	* out-of-sync. Make sure to update the required fields
3716	* before using them.
3717	*
3718	* @remarks No-long-jump zone!!!
3719	*/
3720	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3721	{
3722	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3723	{
3724	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3725	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3726	#endif
3727	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3728	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3729	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3730	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3731	#else
3732	/* 64-bit host or hybrid host. */
3733	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3734	#endif
3735	}
3736	else
3737	{
3738	/* Guest is not in long mode, use the 32-bit handler. */
3739	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3740	}
3741	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3742	return VINF_SUCCESS;
3743	}
3744
3745
3746	/**
3747	* Wrapper for running the guest code in VT-x.
3748	*
3749	* @returns VBox strict status code.
3750	* @param pVM Pointer to the VM.
3751	* @param pVCpu Pointer to the VMCPU.
3752	* @param pCtx Pointer to the guest-CPU context.
3753	*
3754	* @remarks No-long-jump zone!!!
3755	*/
3756	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3757	{
3758	/*
3759	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3760	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3761	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3762	*/
3763	#ifdef VBOX_WITH_KERNEL_USING_XMM
3764	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3765	#else
3766	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3767	#endif
3768	}
3769
3770
3771	/**
3772	* Report world-switch error and dump some useful debug info.
3773	*
3774	* @param pVM Pointer to the VM.
3775	* @param pVCpu Pointer to the VMCPU.
3776	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3777	* @param pCtx Pointer to the guest-CPU context.
3778	* @param pVmxTransient Pointer to the VMX transient structure (only
3779	* exitReason updated).
3780	*/
3781	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3782	{
3783	Assert(pVM);
3784	Assert(pVCpu);
3785	Assert(pCtx);
3786	Assert(pVmxTransient);
3787	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3788
3789	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3790	switch (rcVMRun)
3791	{
3792	case VERR_VMX_INVALID_VMXON_PTR:
3793	AssertFailed();
3794	break;
3795	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3796	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3797	{
3798	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3799	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3800	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3801	AssertRC(rc);
3802
3803	#ifdef VBOX_STRICT
3804	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3805	pVmxTransient->uExitReason));
3806	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3807	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3808	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3809	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3810	else
3811	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3812
3813	/* VMX control bits. */
3814	uint32_t u32Val;
3815	uint64_t u64Val;
3816	RTHCUINTREG uHCReg;
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3845	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3846	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3847	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3848	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3849	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3850	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3851	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3852	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3853	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3854	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3855	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3856	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3857	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3858	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3859
3860	/* Guest bits. */
3861	RTGCUINTREG uGCReg;
3862	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3863	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3864	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3865	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3866	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3867	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3868	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3869	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3870
3871	/* Host bits. */
3872	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3873	Log(("Host CR0 %#RHr\n", uHCReg));
3874	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3875	Log(("Host CR3 %#RHr\n", uHCReg));
3876	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3877	Log(("Host CR4 %#RHr\n", uHCReg));
3878
3879	RTGDTR HostGdtr;
3880	PCX86DESCHC pDesc;
3881	ASMGetGDTR(&HostGdtr);
3882	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3883	Log(("Host CS %#08x\n", u32Val));
3884	if (u32Val < HostGdtr.cbGdt)
3885	{
3886	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3887	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3888	}
3889
3890	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3891	Log(("Host DS %#08x\n", u32Val));
3892	if (u32Val < HostGdtr.cbGdt)
3893	{
3894	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3895	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3896	}
3897
3898	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3899	Log(("Host ES %#08x\n", u32Val));
3900	if (u32Val < HostGdtr.cbGdt)
3901	{
3902	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3903	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3904	}
3905
3906	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3907	Log(("Host FS %#08x\n", u32Val));
3908	if (u32Val < HostGdtr.cbGdt)
3909	{
3910	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3911	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3912	}
3913
3914	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3915	Log(("Host GS %#08x\n", u32Val));
3916	if (u32Val < HostGdtr.cbGdt)
3917	{
3918	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3919	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3920	}
3921
3922	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3923	Log(("Host SS %#08x\n", u32Val));
3924	if (u32Val < HostGdtr.cbGdt)
3925	{
3926	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3927	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3928	}
3929
3930	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3931	Log(("Host TR %#08x\n", u32Val));
3932	if (u32Val < HostGdtr.cbGdt)
3933	{
3934	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3935	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3936	}
3937
3938	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3939	Log(("Host TR Base %#RHv\n", uHCReg));
3940	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3941	Log(("Host GDTR Base %#RHv\n", uHCReg));
3942	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3943	Log(("Host IDTR Base %#RHv\n", uHCReg));
3944	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3945	Log(("Host SYSENTER CS %#08x\n", u32Val));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3947	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3949	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3950	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3951	Log(("Host RSP %#RHv\n", uHCReg));
3952	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3953	Log(("Host RIP %#RHv\n", uHCReg));
3954	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3955	if (HMVMX_IS_64BIT_HOST_MODE())
3956	{
3957	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3958	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3959	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3960	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3961	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3962	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3963	}
3964	# endif
3965	#endif /* VBOX_STRICT */
3966	break;
3967	}
3968
3969	default:
3970	/* Impossible */
3971	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3972	break;
3973	}
3974	NOREF(pVM);
3975	}
3976
3977
3978	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3979	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3980	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3981	#endif
3982
3983	#ifdef VBOX_STRICT
3984	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3985	{
3986	switch (idxField)
3987	{
3988	case VMX_VMCS_GUEST_RIP:
3989	case VMX_VMCS_GUEST_RSP:
3990	case VMX_VMCS_GUEST_RFLAGS:
3991	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3992	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3993	case VMX_VMCS_GUEST_CR0:
3994	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3995	case VMX_VMCS_GUEST_CR4:
3996	case VMX_VMCS_GUEST_DR7:
3997	case VMX_VMCS32_GUEST_SYSENTER_CS:
3998	case VMX_VMCS_GUEST_SYSENTER_EIP:
3999	case VMX_VMCS_GUEST_SYSENTER_ESP:
4000	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4001	case VMX_VMCS_GUEST_GDTR_BASE:
4002	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4003	case VMX_VMCS_GUEST_IDTR_BASE:
4004	case VMX_VMCS16_GUEST_FIELD_CS:
4005	case VMX_VMCS32_GUEST_CS_LIMIT:
4006	case VMX_VMCS_GUEST_CS_BASE:
4007	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4008	case VMX_VMCS16_GUEST_FIELD_DS:
4009	case VMX_VMCS32_GUEST_DS_LIMIT:
4010	case VMX_VMCS_GUEST_DS_BASE:
4011	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4012	case VMX_VMCS16_GUEST_FIELD_ES:
4013	case VMX_VMCS32_GUEST_ES_LIMIT:
4014	case VMX_VMCS_GUEST_ES_BASE:
4015	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4016	case VMX_VMCS16_GUEST_FIELD_FS:
4017	case VMX_VMCS32_GUEST_FS_LIMIT:
4018	case VMX_VMCS_GUEST_FS_BASE:
4019	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4020	case VMX_VMCS16_GUEST_FIELD_GS:
4021	case VMX_VMCS32_GUEST_GS_LIMIT:
4022	case VMX_VMCS_GUEST_GS_BASE:
4023	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4024	case VMX_VMCS16_GUEST_FIELD_SS:
4025	case VMX_VMCS32_GUEST_SS_LIMIT:
4026	case VMX_VMCS_GUEST_SS_BASE:
4027	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4028	case VMX_VMCS16_GUEST_FIELD_LDTR:
4029	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4030	case VMX_VMCS_GUEST_LDTR_BASE:
4031	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4032	case VMX_VMCS16_GUEST_FIELD_TR:
4033	case VMX_VMCS32_GUEST_TR_LIMIT:
4034	case VMX_VMCS_GUEST_TR_BASE:
4035	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4036	case VMX_VMCS32_RO_EXIT_REASON:
4037	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4038	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4039	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4040	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4041	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4042	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4043	case VMX_VMCS32_RO_IDT_INFO:
4044	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4045	case VMX_VMCS_GUEST_CR3:
4046	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4047	return true;
4048	}
4049	return false;
4050	}
4051
4052	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4053	{
4054	switch (idxField)
4055	{
4056	case VMX_VMCS_GUEST_LDTR_BASE:
4057	case VMX_VMCS_GUEST_TR_BASE:
4058	case VMX_VMCS_GUEST_GDTR_BASE:
4059	case VMX_VMCS_GUEST_IDTR_BASE:
4060	case VMX_VMCS_GUEST_SYSENTER_EIP:
4061	case VMX_VMCS_GUEST_SYSENTER_ESP:
4062	case VMX_VMCS_GUEST_CR0:
4063	case VMX_VMCS_GUEST_CR4:
4064	case VMX_VMCS_GUEST_CR3:
4065	case VMX_VMCS_GUEST_DR7:
4066	case VMX_VMCS_GUEST_RIP:
4067	case VMX_VMCS_GUEST_RSP:
4068	case VMX_VMCS_GUEST_CS_BASE:
4069	case VMX_VMCS_GUEST_DS_BASE:
4070	case VMX_VMCS_GUEST_ES_BASE:
4071	case VMX_VMCS_GUEST_FS_BASE:
4072	case VMX_VMCS_GUEST_GS_BASE:
4073	case VMX_VMCS_GUEST_SS_BASE:
4074	return true;
4075	}
4076	return false;
4077	}
4078	#endif /* VBOX_STRICT */
4079
4080	/**
4081	* Executes the specified handler in 64-bit mode.
4082	*
4083	* @returns VBox status code.
4084	* @param pVM Pointer to the VM.
4085	* @param pVCpu Pointer to the VMCPU.
4086	* @param pCtx Pointer to the guest CPU context.
4087	* @param pfnHandler Pointer to the RC handler function.
4088	* @param cbParam Number of parameters.
4089	* @param paParam Array of 32-bit parameters.
4090	*/
4091	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4092	uint32_t *paParam)
4093	{
4094	int rc, rc2;
4095	PHMGLOBLCPUINFO pCpu;
4096	RTHCPHYS HCPhysCpuPage;
4097	RTHCUINTREG uOldEFlags;
4098
4099	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4100	Assert(pfnHandler);
4101	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4102	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4103
4104	#ifdef VBOX_STRICT
4105	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4106	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4107
4108	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4109	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4110	#endif
4111
4112	/* Disable interrupts. */
4113	uOldEFlags = ASMIntDisableFlags();
4114
4115	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4116	RTCPUID idHostCpu = RTMpCpuId();
4117	CPUMR0SetLApic(pVM, idHostCpu);
4118	#endif
4119
4120	pCpu = HMR0GetCurrentCpu();
4121	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4122
4123	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4124	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4125
4126	/* Leave VMX Root Mode. */
4127	VMXDisable();
4128
4129	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4130
4131	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4132	CPUMSetHyperEIP(pVCpu, pfnHandler);
4133	for (int i = (int)cbParam - 1; i >= 0; i--)
4134	CPUMPushHyper(pVCpu, paParam[i]);
4135
4136	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4137
4138	/* Call the switcher. */
4139	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4140	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4141
4142	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4143	/* Make sure the VMX instructions don't cause #UD faults. */
4144	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4145
4146	/* Re-enter VMX Root Mode */
4147	rc2 = VMXEnable(HCPhysCpuPage);
4148	if (RT_FAILURE(rc2))
4149	{
4150	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4151	ASMSetFlags(uOldEFlags);
4152	return rc2;
4153	}
4154
4155	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4156	AssertRC(rc2);
4157	Assert(!(ASMGetFlags() & X86_EFL_IF));
4158	ASMSetFlags(uOldEFlags);
4159	return rc;
4160	}
4161
4162
4163	/**
4164	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4165	* supporting 64-bit guests.
4166	*
4167	* @returns VBox status code.
4168	* @param fResume Whether to VMLAUNCH or VMRESUME.
4169	* @param pCtx Pointer to the guest-CPU context.
4170	* @param pCache Pointer to the VMCS cache.
4171	* @param pVM Pointer to the VM.
4172	* @param pVCpu Pointer to the VMCPU.
4173	*/
4174	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4175	{
4176	uint32_t aParam[6];
4177	PHMGLOBLCPUINFO pCpu = NULL;
4178	RTHCPHYS HCPhysCpuPage = 0;
4179	int rc = VERR_INTERNAL_ERROR_5;
4180
4181	pCpu = HMR0GetCurrentCpu();
4182	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4183
4184	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4185	pCache->uPos = 1;
4186	pCache->interPD = PGMGetInterPaeCR3(pVM);
4187	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4188	#endif
4189
4190	#ifdef VBOX_STRICT
4191	pCache->TestIn.HCPhysCpuPage = 0;
4192	pCache->TestIn.HCPhysVmcs = 0;
4193	pCache->TestIn.pCache = 0;
4194	pCache->TestOut.HCPhysVmcs = 0;
4195	pCache->TestOut.pCache = 0;
4196	pCache->TestOut.pCtx = 0;
4197	pCache->TestOut.eflags = 0;
4198	#endif
4199
4200	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4201	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4202	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4203	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4204	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4205	aParam[5] = 0;
4206
4207	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4208	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4209	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4210	#endif
4211	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4212
4213	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4214	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4215	Assert(pCtx->dr[4] == 10);
4216	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4217	#endif
4218
4219	#ifdef VBOX_STRICT
4220	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4221	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4222	pVCpu->hm.s.vmx.HCPhysVmcs));
4223	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4224	pCache->TestOut.HCPhysVmcs));
4225	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4226	pCache->TestOut.pCache));
4227	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4228	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4229	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4230	pCache->TestOut.pCtx));
4231	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4232	#endif
4233	return rc;
4234	}
4235
4236
4237	/**
4238	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4239	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4240	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4241	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4242	*
4243	* @returns VBox status code.
4244	* @param pVM Pointer to the VM.
4245	* @param pVCpu Pointer to the VMCPU.
4246	*/
4247	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4248	{
4249	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4250	{ \
4251	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4252	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4253	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4254	}
4255
4256	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4257	{ \
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4261	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4262	}
4263
4264	AssertPtr(pVM);
4265	AssertPtr(pVCpu);
4266	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4267
4268	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4269	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4270	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4271	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4272	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4273	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4274	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4275	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4276	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4277
4278	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4279	#if 0
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4289	#endif
4290
4291	/* 32-bit guest-state fields. */
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4296	/* Unused 32-bit guest-state fields. */
4297	#if 0
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4301	#endif
4302
4303	/* Natural width guest-state fields. */
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4320	/* Unused natural width guest-state fields. */
4321	#if 0
4322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4324	#endif
4325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4327
4328	if (pVM->hm.s.fNestedPaging)
4329	{
4330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4332	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4333	}
4334	else
4335	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4336
4337	#undef VMXLOCAL_INIT_VMCS_SELREG
4338	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4339	return VINF_SUCCESS;
4340	}
4341
4342
4343	/**
4344	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4345	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4346	* darwin, running 64-bit guests).
4347	*
4348	* @returns VBox status code.
4349	* @param pVCpu Pointer to the VMCPU.
4350	* @param idxField The VMCS field encoding.
4351	* @param u64Val 16, 32 or 64 bits value.
4352	*/
4353	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4354	{
4355	int rc;
4356	switch (idxField)
4357	{
4358	/*
4359	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4360	*/
4361	/* 64-bit Control fields. */
4362	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4363	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4364	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4365	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4366	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4367	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4368	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4369	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4370	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4371	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4372	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4373	case VMX_VMCS64_CTRL_EPTP_FULL:
4374	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4375	/* 64-bit Read-only data fields. */
4376	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4377	/* 64-bit Guest-state fields. */
4378	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4379	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4380	case VMX_VMCS64_GUEST_PAT_FULL:
4381	case VMX_VMCS64_GUEST_EFER_FULL:
4382	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4383	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4384	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4385	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4386	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4387	/* 64-bit Host-state fields. */
4388	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4389	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4390	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4391	{
4392	rc = VMXWriteVmcs32(idxField, u64Val);
4393	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4394	break;
4395	}
4396
4397	/*
4398	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4399	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4400	*/
4401	/* Natural-width Guest-state fields. */
4402	case VMX_VMCS_GUEST_CR0:
4403	case VMX_VMCS_GUEST_CR3:
4404	case VMX_VMCS_GUEST_CR4:
4405	case VMX_VMCS_GUEST_ES_BASE:
4406	case VMX_VMCS_GUEST_CS_BASE:
4407	case VMX_VMCS_GUEST_SS_BASE:
4408	case VMX_VMCS_GUEST_DS_BASE:
4409	case VMX_VMCS_GUEST_FS_BASE:
4410	case VMX_VMCS_GUEST_GS_BASE:
4411	case VMX_VMCS_GUEST_LDTR_BASE:
4412	case VMX_VMCS_GUEST_TR_BASE:
4413	case VMX_VMCS_GUEST_GDTR_BASE:
4414	case VMX_VMCS_GUEST_IDTR_BASE:
4415	case VMX_VMCS_GUEST_DR7:
4416	case VMX_VMCS_GUEST_RSP:
4417	case VMX_VMCS_GUEST_RIP:
4418	case VMX_VMCS_GUEST_RFLAGS:
4419	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4420	case VMX_VMCS_GUEST_SYSENTER_ESP:
4421	case VMX_VMCS_GUEST_SYSENTER_EIP:
4422	{
4423	if ((u64Val >> 32ULL) == 0)
4424	{
4425	/* If this field is 64-bit, VT-x will zero out the top bits. */
4426	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4427	}
4428	else
4429	{
4430	/* Assert that only the 32->64 switcher case should ever come here. */
4431	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4432	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4433	}
4434	break;
4435	}
4436
4437	default:
4438	{
4439	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4440	rc = VERR_INVALID_PARAMETER;
4441	break;
4442	}
4443	}
4444	AssertRCReturn(rc, rc);
4445	return rc;
4446	}
4447
4448
4449	/**
4450	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4451	* hosts (except darwin) for 64-bit guests.
4452	*
4453	* @param pVCpu Pointer to the VMCPU.
4454	* @param idxField The VMCS field encoding.
4455	* @param u64Val 16, 32 or 64 bits value.
4456	*/
4457	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4458	{
4459	AssertPtr(pVCpu);
4460	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4461
4462	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4463	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4464
4465	/* Make sure there are no duplicates. */
4466	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4467	{
4468	if (pCache->Write.aField[i] == idxField)
4469	{
4470	pCache->Write.aFieldVal[i] = u64Val;
4471	return VINF_SUCCESS;
4472	}
4473	}
4474
4475	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4476	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4477	pCache->Write.cValidEntries++;
4478	return VINF_SUCCESS;
4479	}
4480
4481
4482	/**
4483	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4484	*
4485	* @param pVCpu Pointer to the VMCPU.
4486	* @param pCache Pointer to the VMCS cache.
4487	*
4488	* @remarks No-long-jump zone!!!
4489	*/
4490	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4491	{
4492	AssertPtr(pCache);
4493	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4494	{
4495	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4496	AssertRC(rc);
4497	}
4498	pCache->Write.cValidEntries = 0;
4499	}
4500
4501
4502	/**
4503	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4504	*
4505	* @param pVCpu Pointer to the VMCPU.
4506	* @param pCache Pointer to the VMCS cache.
4507	*
4508	* @remarks No-long-jump zone!!!
4509	*/
4510	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4511	{
4512	AssertPtr(pCache);
4513	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4514	{
4515	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4516	AssertRC(rc);
4517	}
4518	}
4519	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4520
4521
4522	/**
4523	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4524	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4525	* timer.
4526	*
4527	* @returns VBox status code.
4528	* @param pVCpu Pointer to the VMCPU.
4529	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4530	* out-of-sync. Make sure to update the required fields
4531	* before using them.
4532	* @remarks No-long-jump zone!!!
4533	*/
4534	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4535	{
4536	int rc = VERR_INTERNAL_ERROR_5;
4537	bool fOffsettedTsc = false;
4538	PVM pVM = pVCpu->CTX_SUFF(pVM);
4539	if (pVM->hm.s.vmx.fUsePreemptTimer)
4540	{
4541	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4542
4543	/* Make sure the returned values have sane upper and lower boundaries. */
4544	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4545	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4546	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4547	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4548
4549	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4550	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4551	}
4552	else
4553	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4554
4555	if (fOffsettedTsc)
4556	{
4557	uint64_t u64CurTSC = ASMReadTSC();
4558	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4559	{
4560	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4561	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4562
4563	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4564	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4565	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4566	}
4567	else
4568	{
4569	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4570	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4571	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4572	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4573	}
4574	}
4575	else
4576	{
4577	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4578	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4579	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4580	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4581	}
4582	}
4583
4584
4585	/**
4586	* Determines if an exception is a contributory exception. Contributory
4587	* exceptions are ones which can cause double-faults. Page-fault is
4588	* intentionally not included here as it's a conditional contributory exception.
4589	*
4590	* @returns true if the exception is contributory, false otherwise.
4591	* @param uVector The exception vector.
4592	*/
4593	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4594	{
4595	switch (uVector)
4596	{
4597	case X86_XCPT_GP:
4598	case X86_XCPT_SS:
4599	case X86_XCPT_NP:
4600	case X86_XCPT_TS:
4601	case X86_XCPT_DE:
4602	return true;
4603	default:
4604	break;
4605	}
4606	return false;
4607	}
4608
4609
4610	/**
4611	* Sets an event as a pending event to be injected into the guest.
4612	*
4613	* @param pVCpu Pointer to the VMCPU.
4614	* @param u32IntrInfo The VM-entry interruption-information field.
4615	* @param cbInstr The VM-entry instruction length in bytes (for software
4616	* interrupts, exceptions and privileged software
4617	* exceptions).
4618	* @param u32ErrCode The VM-entry exception error code.
4619	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4620	* page-fault.
4621	*/
4622	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4623	RTGCUINTPTR GCPtrFaultAddress)
4624	{
4625	Assert(!pVCpu->hm.s.Event.fPending);
4626	pVCpu->hm.s.Event.fPending = true;
4627	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4628	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4629	pVCpu->hm.s.Event.cbInstr = cbInstr;
4630	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4631	}
4632
4633
4634	/**
4635	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4636	*
4637	* @param pVCpu Pointer to the VMCPU.
4638	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4639	* out-of-sync. Make sure to update the required fields
4640	* before using them.
4641	*/
4642	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4643	{
4644	/* Inject the double-fault. */
4645	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4646	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4647	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4648	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4649	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4650	}
4651
4652
4653	/**
4654	* Handle a condition that occurred while delivering an event through the guest
4655	* IDT.
4656	*
4657	* @returns VBox status code (informational error codes included).
4658	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4659	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4660	* continue execution of the guest which will delivery the #DF.
4661	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4662	*
4663	* @param pVCpu Pointer to the VMCPU.
4664	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4665	* out-of-sync. Make sure to update the required fields
4666	* before using them.
4667	* @param pVmxTransient Pointer to the VMX transient structure.
4668	*
4669	* @remarks No-long-jump zone!!!
4670	*/
4671	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4672	{
4673	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4674	AssertRC(rc);
4675	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4676	{
4677	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4678	AssertRCReturn(rc, rc);
4679
4680	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4681	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4682	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4683
4684	typedef enum
4685	{
4686	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4687	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4688	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4689	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4690	} VMXREFLECTXCPT;
4691
4692	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4693	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4694	RTGCUINTPTR GCPtrFaultAddress = 0;
4695	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4696	{
4697	enmReflect = VMXREFLECTXCPT_XCPT;
4698	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4699	&& uExitVector == X86_XCPT_PF)
4700	{
4701	GCPtrFaultAddress = pMixedCtx->cr2;
4702	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4703	}
4704	else if ( uExitVector == X86_XCPT_PF
4705	&& uIdtVector == X86_XCPT_PF)
4706	{
4707	pVmxTransient->fVectoringPF = true;
4708	GCPtrFaultAddress = pMixedCtx->cr2;
4709	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4710	}
4711	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4712	&& hmR0VmxIsContributoryXcpt(uExitVector)
4713	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4714	\|\| uIdtVector == X86_XCPT_PF))
4715	{
4716	enmReflect = VMXREFLECTXCPT_DF;
4717	}
4718	else if (uIdtVector == X86_XCPT_DF)
4719	enmReflect = VMXREFLECTXCPT_TF;
4720	}
4721	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4722	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4723	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4724	{
4725	/*
4726	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4727	* (whatever they are) as they reoccur when restarting the instruction.
4728	*/
4729	enmReflect = VMXREFLECTXCPT_XCPT;
4730	}
4731
4732	switch (enmReflect)
4733	{
4734	case VMXREFLECTXCPT_XCPT:
4735	{
4736	uint32_t u32ErrCode = 0;
4737	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4738	{
4739	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4740	AssertRCReturn(rc, rc);
4741	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4742	}
4743	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4744	0 /* cbInstr */, u32ErrCode, GCPtrFaultAddress);
4745	rc = VINF_SUCCESS;
4746	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4747	break;
4748	}
4749
4750	case VMXREFLECTXCPT_DF:
4751	{
4752	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4753	rc = VINF_VMX_DOUBLE_FAULT;
4754	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4755	uExitVector));
4756	break;
4757	}
4758
4759	case VMXREFLECTXCPT_TF:
4760	{
4761	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4762	rc = VINF_EM_RESET;
4763	break;
4764	}
4765
4766	default:
4767	Assert(rc == VINF_SUCCESS);
4768	break;
4769	}
4770	}
4771	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4772	return rc;
4773	}
4774
4775
4776	/**
4777	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4778	*
4779	* @returns VBox status code.
4780	* @param pVCpu Pointer to the VMCPU.
4781	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4782	* out-of-sync. Make sure to update the required fields
4783	* before using them.
4784	*
4785	* @remarks No-long-jump zone!!!
4786	*/
4787	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4788	{
4789	int rc = VINF_SUCCESS;
4790	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4791	{
4792	RTGCUINTREG uVal = 0;
4793	RTCCUINTREG uShadow = 0;
4794	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4795	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4796	AssertRCReturn(rc, rc);
4797	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4798	CPUMSetGuestCR0(pVCpu, uVal);
4799	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4800	}
4801	return rc;
4802	}
4803
4804
4805	/**
4806	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4807	*
4808	* @returns VBox status code.
4809	* @param pVCpu Pointer to the VMCPU.
4810	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4811	* out-of-sync. Make sure to update the required fields
4812	* before using them.
4813	*
4814	* @remarks No-long-jump zone!!!
4815	*/
4816	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4817	{
4818	int rc = VINF_SUCCESS;
4819	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4820	{
4821	RTGCUINTREG uVal = 0;
4822	RTCCUINTREG uShadow = 0;
4823	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4824	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4825	AssertRCReturn(rc, rc);
4826	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4827	CPUMSetGuestCR4(pVCpu, uVal);
4828	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4829	}
4830	return rc;
4831	}
4832
4833
4834	/**
4835	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4836	*
4837	* @returns VBox status code.
4838	* @param pVCpu Pointer to the VMCPU.
4839	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4840	* out-of-sync. Make sure to update the required fields
4841	* before using them.
4842	*
4843	* @remarks No-long-jump zone!!!
4844	*/
4845	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4846	{
4847	int rc = VINF_SUCCESS;
4848	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4849	{
4850	RTGCUINTREG uVal = 0;
4851	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4852	AssertRCReturn(rc, rc);
4853	pMixedCtx->rip = uVal;
4854	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4855	}
4856	return rc;
4857	}
4858
4859
4860	/**
4861	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4862	*
4863	* @returns VBox status code.
4864	* @param pVCpu Pointer to the VMCPU.
4865	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4866	* out-of-sync. Make sure to update the required fields
4867	* before using them.
4868	*
4869	* @remarks No-long-jump zone!!!
4870	*/
4871	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4872	{
4873	int rc = VINF_SUCCESS;
4874	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4875	{
4876	RTGCUINTREG uVal = 0;
4877	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4878	AssertRCReturn(rc, rc);
4879	pMixedCtx->rsp = uVal;
4880	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4881	}
4882	return rc;
4883	}
4884
4885
4886	/**
4887	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4888	*
4889	* @returns VBox status code.
4890	* @param pVCpu Pointer to the VMCPU.
4891	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4892	* out-of-sync. Make sure to update the required fields
4893	* before using them.
4894	*
4895	* @remarks No-long-jump zone!!!
4896	*/
4897	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4898	{
4899	int rc = VINF_SUCCESS;
4900	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4901	{
4902	RTGCUINTREG uVal = 0;
4903	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4904	AssertRCReturn(rc, rc);
4905	pMixedCtx->rflags.u64 = uVal;
4906
4907	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4908	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4909	{
4910	PVM pVM = pVCpu->CTX_SUFF(pVM);
4911	Assert(pVM->hm.s.vmx.pRealModeTSS);
4912	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4913	pMixedCtx->eflags.Bits.u1VM = 0;
4914	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4915	}
4916
4917	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4918	}
4919	return rc;
4920	}
4921
4922
4923	/**
4924	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4925	* guest-CPU context.
4926	*/
4927	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4928	{
4929	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4930	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4931	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4932	return rc;
4933	}
4934
4935
4936	/**
4937	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4938	* from the guest-state area in the VMCS.
4939	*
4940	* @param pVCpu Pointer to the VMCPU.
4941	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4942	* out-of-sync. Make sure to update the required fields
4943	* before using them.
4944	*
4945	* @remarks No-long-jump zone!!!
4946	*/
4947	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4948	{
4949	uint32_t uIntrState = 0;
4950	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4951	AssertRC(rc);
4952
4953	if (!uIntrState)
4954	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4955	else
4956	{
4957	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4958	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4959	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4960	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4961	AssertRC(rc);
4962	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4963	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4964	}
4965	}
4966
4967
4968	/**
4969	* Saves the guest's activity state.
4970	*
4971	* @returns VBox status code.
4972	* @param pVCpu Pointer to the VMCPU.
4973	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4974	* out-of-sync. Make sure to update the required fields
4975	* before using them.
4976	*
4977	* @remarks No-long-jump zone!!!
4978	*/
4979	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4980	{
4981	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4982	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4983	return VINF_SUCCESS;
4984	}
4985
4986
4987	/**
4988	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4989	* the current VMCS into the guest-CPU context.
4990	*
4991	* @returns VBox status code.
4992	* @param pVCpu Pointer to the VMCPU.
4993	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4994	* out-of-sync. Make sure to update the required fields
4995	* before using them.
4996	*
4997	* @remarks No-long-jump zone!!!
4998	*/
4999	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5000	{
5001	int rc = VINF_SUCCESS;
5002	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5003	{
5004	uint32_t u32Val = 0;
5005	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5006	pMixedCtx->SysEnter.cs = u32Val;
5007	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5008	}
5009
5010	RTGCUINTREG uGCVal = 0;
5011	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5012	{
5013	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5014	pMixedCtx->SysEnter.eip = uGCVal;
5015	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5016	}
5017	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5018	{
5019	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5020	pMixedCtx->SysEnter.esp = uGCVal;
5021	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5022	}
5023	return rc;
5024	}
5025
5026
5027	/**
5028	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5029	* context.
5030	*
5031	* @returns VBox status code.
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5034	* out-of-sync. Make sure to update the required fields
5035	* before using them.
5036	*
5037	* @remarks No-long-jump zone!!!
5038	*/
5039	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5040	{
5041	RTGCUINTREG uVal = 0;
5042	int rc = VINF_SUCCESS;
5043	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5044	{
5045	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5046	pMixedCtx->fs.u64Base = uVal;
5047	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5048	}
5049	return rc;
5050	}
5051
5052
5053	/**
5054	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5055	* context.
5056	*
5057	* @returns VBox status code.
5058	* @param pVCpu Pointer to the VMCPU.
5059	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5060	* out-of-sync. Make sure to update the required fields
5061	* before using them.
5062	*
5063	* @remarks No-long-jump zone!!!
5064	*/
5065	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5066	{
5067	RTGCUINTREG uVal = 0;
5068	int rc = VINF_SUCCESS;
5069	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5070	{
5071	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5072	pMixedCtx->gs.u64Base = uVal;
5073	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5074	}
5075	return rc;
5076	}
5077
5078
5079	/**
5080	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5081	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5082	*
5083	* @returns VBox status code.
5084	* @param pVCpu Pointer to the VMCPU.
5085	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5086	* out-of-sync. Make sure to update the required fields
5087	* before using them.
5088	*
5089	* @remarks No-long-jump zone!!!
5090	*/
5091	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5092	{
5093	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5094	return VINF_SUCCESS;
5095
5096	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5097	{
5098	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5099	pMsr += i;
5100	switch (pMsr->u32IndexMSR)
5101	{
5102	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5103	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5104	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5105	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5106	#if 0
5107	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5108	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5109	#endif
5110	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5111	default:
5112	{
5113	AssertFailed();
5114	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5115	}
5116	}
5117	}
5118	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5119	return VINF_SUCCESS;
5120	}
5121
5122
5123	/**
5124	* Saves the guest control registers from the current VMCS into the guest-CPU
5125	* context.
5126	*
5127	* @returns VBox status code.
5128	* @param pVCpu Pointer to the VMCPU.
5129	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5130	* out-of-sync. Make sure to update the required fields
5131	* before using them.
5132	*
5133	* @remarks No-long-jump zone!!!
5134	*/
5135	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5136	{
5137	RTGCUINTREG uVal = 0;
5138	RTGCUINTREG uShadow = 0;
5139	int rc = VINF_SUCCESS;
5140
5141	/* Guest CR0. Guest FPU. */
5142	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5143
5144	/* Guest CR4. */
5145	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5146	AssertRCReturn(rc, rc);
5147
5148	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5149	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5150	{
5151	PVM pVM = pVCpu->CTX_SUFF(pVM);
5152	if ( pVM->hm.s.fNestedPaging
5153	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5154	{
5155	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5156	if (pMixedCtx->cr3 != uVal)
5157	{
5158	CPUMSetGuestCR3(pVCpu, uVal);
5159	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5160	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5161	}
5162
5163	/* We require EFER to check PAE mode. */
5164	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5165
5166	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5167	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5168	{
5169	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5170	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5171	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5172	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5173	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5174	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5175	}
5176	AssertRCReturn(rc, rc);
5177	}
5178
5179	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5180	}
5181	return rc;
5182	}
5183
5184
5185	/**
5186	* Reads a guest segment register from the current VMCS into the guest-CPU
5187	* context.
5188	*
5189	* @returns VBox status code.
5190	* @param pVCpu Pointer to the VMCPU.
5191	* @param idxSel Index of the selector in the VMCS.
5192	* @param idxLimit Index of the segment limit in the VMCS.
5193	* @param idxBase Index of the segment base in the VMCS.
5194	* @param idxAccess Index of the access rights of the segment in the VMCS.
5195	* @param pSelReg Pointer to the segment selector.
5196	*
5197	* @remarks No-long-jump zone!!!
5198	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5199	* as that takes care of whether to read from the VMCS cache or not.
5200	*/
5201	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5202	PCPUMSELREG pSelReg)
5203	{
5204	uint32_t u32Val = 0;
5205	int rc = VMXReadVmcs32(idxSel, &u32Val);
5206	pSelReg->Sel = (uint16_t)u32Val;
5207	pSelReg->ValidSel = (uint16_t)u32Val;
5208	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5209
5210	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5211	pSelReg->u32Limit = u32Val;
5212
5213	RTGCUINTREG uGCVal = 0;
5214	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5215	pSelReg->u64Base = uGCVal;
5216
5217	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5218	pSelReg->Attr.u = u32Val;
5219	AssertRCReturn(rc, rc);
5220
5221	/*
5222	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5223	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5224	*/
5225	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5226	{
5227	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5228	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5229	}
5230	return rc;
5231	}
5232
5233
5234	/**
5235	* Saves the guest segment registers from the current VMCS into the guest-CPU
5236	* context.
5237	*
5238	* @returns VBox status code.
5239	* @param pVCpu Pointer to the VMCPU.
5240	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5241	* out-of-sync. Make sure to update the required fields
5242	* before using them.
5243	*
5244	* @remarks No-long-jump zone!!!
5245	*/
5246	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5247	{
5248	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5249	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5250	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5251	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5252	#else
5253	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5254	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5255	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5256	#endif
5257
5258	int rc = VINF_SUCCESS;
5259
5260	/* Guest segment registers. */
5261	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5262	{
5263	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5264	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5265	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5266	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5267	rc \|= VMXLOCAL_READ_SEG(ES, es);
5268	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5269	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5270	AssertRCReturn(rc, rc);
5271
5272	/* Restore segment attributes for real-on-v86 mode hack. */
5273	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5274	{
5275	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5276	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5277	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5278	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5279	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5280	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5281	}
5282	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5283	}
5284
5285	/* Guest LDTR. */
5286	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5287	{
5288	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5289	AssertRCReturn(rc, rc);
5290	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5291	}
5292
5293	/* Guest GDTR. */
5294	RTGCUINTREG uGCVal = 0;
5295	uint32_t u32Val = 0;
5296	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5297	{
5298	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5299	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5300	pMixedCtx->gdtr.pGdt = uGCVal;
5301	pMixedCtx->gdtr.cbGdt = u32Val;
5302	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5303	}
5304
5305	/* Guest IDTR. */
5306	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5307	{
5308	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5309	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5310	pMixedCtx->idtr.pIdt = uGCVal;
5311	pMixedCtx->idtr.cbIdt = u32Val;
5312	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5313	}
5314
5315	/* Guest TR. */
5316	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5317	{
5318	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5319
5320	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5321	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5322	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5323	AssertRCReturn(rc, rc);
5324	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5325	}
5326	return rc;
5327	}
5328
5329
5330	/**
5331	* Saves the guest debug registers from the current VMCS into the guest-CPU
5332	* context.
5333	*
5334	* @returns VBox status code.
5335	* @param pVCpu Pointer to the VMCPU.
5336	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5337	* out-of-sync. Make sure to update the required fields
5338	* before using them.
5339	*
5340	* @remarks No-long-jump zone!!!
5341	*/
5342	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5343	{
5344	int rc = VINF_SUCCESS;
5345	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5346	{
5347	RTGCUINTREG uVal;
5348	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5349	pMixedCtx->dr[7] = uVal;
5350
5351	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5352	}
5353	return rc;
5354	}
5355
5356
5357	/**
5358	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5359	*
5360	* @returns VBox status code.
5361	* @param pVCpu Pointer to the VMCPU.
5362	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5363	* out-of-sync. Make sure to update the required fields
5364	* before using them.
5365	*
5366	* @remarks No-long-jump zone!!!
5367	*/
5368	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5369	{
5370	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5371	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5372	return VINF_SUCCESS;
5373	}
5374
5375
5376	/**
5377	* Saves the entire guest state from the currently active VMCS into the
5378	* guest-CPU context. This essentially VMREADs all guest-data.
5379	*
5380	* @returns VBox status code.
5381	* @param pVCpu Pointer to the VMCPU.
5382	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5383	* out-of-sync. Make sure to update the required fields
5384	* before using them.
5385	*/
5386	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5387	{
5388	Assert(pVCpu);
5389	Assert(pMixedCtx);
5390
5391	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5392	return VINF_SUCCESS;
5393
5394	VMMRZCallRing3Disable(pVCpu);
5395	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5396	LogFunc(("\n"));
5397
5398	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5399	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5400
5401	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5402	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5403
5404	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5405	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5406
5407	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5408	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5409
5410	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5411	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5412
5413	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5414	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5415
5416	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5417	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5418
5419	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5420	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5421
5422	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5423	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5424
5425	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5426	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5427
5428	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5429	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5430
5431	VMMRZCallRing3Enable(pVCpu);
5432	return rc;
5433	}
5434
5435
5436	/**
5437	* Check per-VM and per-VCPU force flag actions that require us to go back to
5438	* ring-3 for one reason or another.
5439	*
5440	* @returns VBox status code (information status code included).
5441	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5442	* ring-3.
5443	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5444	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5445	* interrupts)
5446	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5447	* all EMTs to be in ring-3.
5448	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5449	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5450	* to the EM loop.
5451	*
5452	* @param pVM Pointer to the VM.
5453	* @param pVCpu Pointer to the VMCPU.
5454	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5455	* out-of-sync. Make sure to update the required fields
5456	* before using them.
5457	*/
5458	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5459	{
5460	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5461
5462	int rc = VERR_INTERNAL_ERROR_5;
5463	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5464	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5465	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5466	{
5467	/* We need the control registers now, make sure the guest-CPU context is updated. */
5468	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5469	AssertRCReturn(rc, rc);
5470
5471	/* Pending HM CR3 sync. */
5472	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5473	{
5474	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5475	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5476	}
5477	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5478	{
5479	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5480	AssertRC(rc);
5481	}
5482
5483	/* Pending PGM C3 sync. */
5484	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5485	{
5486	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5487	if (rc != VINF_SUCCESS)
5488	{
5489	AssertRC(rc);
5490	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5491	return rc;
5492	}
5493	}
5494
5495	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5496	/* -XXX- what was that about single stepping? */
5497	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5498	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5499	{
5500	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5501	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5502	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5503	return rc;
5504	}
5505
5506	/* Pending VM request packets, such as hardware interrupts. */
5507	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5508	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5509	{
5510	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5511	return VINF_EM_PENDING_REQUEST;
5512	}
5513
5514	/* Pending PGM pool flushes. */
5515	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5516	{
5517	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5518	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5519	}
5520
5521	/* Pending DMA requests. */
5522	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5523	{
5524	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5525	return VINF_EM_RAW_TO_R3;
5526	}
5527	}
5528
5529	/* Paranoia. */
5530	Assert(rc != VERR_EM_INTERPRETER);
5531	return VINF_SUCCESS;
5532	}
5533
5534
5535	/**
5536	* Converts any TRPM trap into a pending VMX event. This is typically used when
5537	* entering from ring-3 (not longjmp returns).
5538	*
5539	* @param pVCpu Pointer to the VMCPU.
5540	* @param pCtx Pointer to the guest-CPU context.
5541	*/
5542	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5543	{
5544	if (!TRPMHasTrap(pVCpu))
5545	{
5546	Assert(!pVCpu->hm.s.Event.fPending);
5547	return;
5548	}
5549
5550	uint8_t uVector;
5551	TRPMEVENT enmTrpmEvent;
5552	RTGCUINT uErrCode;
5553	RTGCUINTPTR GCPtrFaultAddress;
5554	uint8_t cbInstr;
5555
5556	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5557	AssertRC(rc);
5558
5559	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5560	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5561	uint32_t u32ErrCode = uErrCode;
5562	if (enmTrpmEvent == TRPM_TRAP)
5563	{
5564	switch (uVector)
5565	{
5566	case X86_XCPT_BP:
5567	case X86_XCPT_OF:
5568	{
5569	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5570	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5571	break;
5572	}
5573
5574	case X86_XCPT_PF:
5575	pCtx->cr2 = GCPtrFaultAddress;
5576	/* no break */
5577	case X86_XCPT_DF:
5578	case X86_XCPT_TS:
5579	case X86_XCPT_NP:
5580	case X86_XCPT_SS:
5581	case X86_XCPT_GP:
5582	case X86_XCPT_AC:
5583	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5584	them which VT-x/VMM pushes to the guest stack. */
5585	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5586	/* no break! */
5587	default:
5588	{
5589	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5590	break;
5591	}
5592	}
5593	}
5594	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5595	{
5596	if (uVector != X86_XCPT_NMI)
5597	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5598	else
5599	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5600	}
5601	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5602	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5603	else
5604	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5605
5606	rc = TRPMResetTrap(pVCpu);
5607	AssertRC(rc);
5608	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5609	u32IntrInfo, enmTrpmEvent, cbInstr, u32ErrCode, GCPtrFaultAddress));
5610	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5611	}
5612
5613
5614	/**
5615	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5616	* VT-x to execute any instruction.
5617	*
5618	* @param pvCpu Pointer to the VMCPU.
5619	*/
5620	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5621	{
5622	if (pVCpu->hm.s.Event.fPending)
5623	{
5624	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5625	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5626	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5627	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5628
5629	/* If a trap was already pending, we did something wrong! */
5630	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5631
5632	TRPMEVENT enmTrapType;
5633	switch (uVectorType)
5634	{
5635	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5636	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5637	enmTrapType = TRPM_HARDWARE_INT;
5638	break;
5639	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5640	enmTrapType = TRPM_SOFTWARE_INT;
5641	break;
5642	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5643	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5644	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5645	enmTrapType = TRPM_TRAP;
5646	break;
5647	default:
5648	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5649	enmTrapType = TRPM_32BIT_HACK;
5650	break;
5651	}
5652
5653	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5654	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5655	AssertRC(rc);
5656
5657	if (fErrorCodeValid)
5658	TRPMSetErrorCode(pVCpu, uErrorCode);
5659	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5660	&& uVector == X86_XCPT_PF)
5661	{
5662	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5663	}
5664	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5665	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5666	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5667	{
5668	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5669	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5670	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5671	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5672	}
5673	pVCpu->hm.s.Event.fPending = false;
5674	}
5675	}
5676
5677
5678	/**
5679	* Does the necessary state syncing before doing a longjmp to ring-3.
5680	*
5681	* @param pVM Pointer to the VM.
5682	* @param pVCpu Pointer to the VMCPU.
5683	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5684	* out-of-sync. Make sure to update the required fields
5685	* before using them.
5686	* @param rcExit The reason for exiting to ring-3. Can be
5687	* VINF_VMM_UNKNOWN_RING3_CALL.
5688	*
5689	* @remarks No-long-jmp zone!!!
5690	*/
5691	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5692	{
5693	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5694	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5695
5696	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5697	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5698	AssertRC(rc);
5699
5700	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5701	if (CPUMIsGuestFPUStateActive(pVCpu))
5702	{
5703	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5704	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5705	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5706	}
5707
5708	/* Restore debug registers if necessary and resync on next R0 reentry. */
5709	if (CPUMIsGuestDebugStateActive(pVCpu))
5710	{
5711	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5712	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5713	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5714	}
5715	else if (CPUMIsHyperDebugStateActive(pVCpu))
5716	{
5717	CPUMR0LoadHostDebugState(pVM, pVCpu);
5718	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5719	}
5720
5721	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5722	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5723	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5724	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5725	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5726	}
5727
5728
5729	/**
5730	* An action requires us to go back to ring-3. This function does the necessary
5731	* steps before we can safely return to ring-3. This is not the same as longjmps
5732	* to ring-3, this is voluntary.
5733	*
5734	* @param pVM Pointer to the VM.
5735	* @param pVCpu Pointer to the VMCPU.
5736	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5737	* out-of-sync. Make sure to update the required fields
5738	* before using them.
5739	* @param rcExit The reason for exiting to ring-3. Can be
5740	* VINF_VMM_UNKNOWN_RING3_CALL.
5741	*/
5742	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5743	{
5744	Assert(pVM);
5745	Assert(pVCpu);
5746	Assert(pMixedCtx);
5747	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5748
5749	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5750	{
5751	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5752	return;
5753	}
5754	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5755	{
5756	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5757	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5758	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5759	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5760	return;
5761	}
5762
5763	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5764	VMMRZCallRing3Disable(pVCpu);
5765	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5766
5767	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5768	hmR0VmxUpdateTRPM(pVCpu);
5769
5770	/* Sync. the guest state. */
5771	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5772	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5773
5774	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5775	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5776	\| CPUM_CHANGED_LDTR
5777	\| CPUM_CHANGED_GDTR
5778	\| CPUM_CHANGED_IDTR
5779	\| CPUM_CHANGED_TR
5780	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5781
5782	/* On our way back from ring-3 the following needs to be done. */
5783	/** @todo This can change with preemption hooks. */
5784	if (rcExit == VINF_EM_RAW_INTERRUPT)
5785	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5786	else
5787	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5788
5789	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5790	VMMRZCallRing3Enable(pVCpu);
5791	}
5792
5793
5794	/**
5795	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5796	* longjump to ring-3 and possibly get preempted.
5797	*
5798	* @param pVCpu Pointer to the VMCPU.
5799	* @param enmOperation The operation causing the ring-3 longjump.
5800	* @param pvUser The user argument (pointer to the possibly
5801	* out-of-date guest-CPU context).
5802	*
5803	* @remarks Must never be called with @a enmOperation ==
5804	* VMMCALLRING3_VM_R0_ASSERTION.
5805	*/
5806	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5807	{
5808	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5809	Assert(pVCpu);
5810	Assert(pvUser);
5811	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5812	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5813
5814	VMMRZCallRing3Disable(pVCpu);
5815	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5816	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5817	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5818	VMMRZCallRing3Enable(pVCpu);
5819	}
5820
5821
5822	/**
5823	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5824	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5825	*
5826	* @returns VBox status code.
5827	* @param pVCpu Pointer to the VMCPU.
5828	*/
5829	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5830	{
5831	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5832	{
5833	/* Enable interrupt-window exiting if it's not in effect already. */
5834	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5835	{
5836	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5837	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5838	AssertRC(rc);
5839	}
5840	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5841	}
5842
5843
5844	/**
5845	* Injects any pending events into the guest if the guest is in a state to
5846	* receive them.
5847	*
5848	* @returns VBox status code (informational status codes included).
5849	* @param pVCpu Pointer to the VMCPU.
5850	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5851	* out-of-sync. Make sure to update the required fields
5852	* before using them.
5853	*/
5854	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5855	{
5856	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5857	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5858	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5859	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5860
5861	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5862	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5863	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5864	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5865
5866	int rc = VINF_SUCCESS;
5867	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5868	{
5869	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5870	bool fInject = true;
5871	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5872	{
5873	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5874	AssertRCReturn(rc, rc);
5875	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5876	if ( fBlockInt
5877	\|\| fBlockSti
5878	\|\| fBlockMovSS)
5879	{
5880	fInject = false;
5881	}
5882	}
5883	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5884	&& ( fBlockMovSS
5885	\|\| fBlockSti))
5886	{
5887	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5888	fInject = false;
5889	}
5890
5891	if (fInject)
5892	{
5893	Log(("Injecting pending event\n"));
5894	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5895	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5896	AssertRCReturn(rc, rc);
5897	pVCpu->hm.s.Event.fPending = false;
5898	}
5899	else
5900	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5901	} /** @todo SMI. SMIs take priority over NMIs. */
5902	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5903	{
5904	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5905	if ( !fBlockMovSS
5906	&& !fBlockSti)
5907	{
5908	Log(("Injecting NMI\n"));
5909	RTGCUINTPTR uIntrInfo;
5910	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5911	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5912	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5913	AssertRCReturn(rc, rc);
5914	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5915	}
5916	else
5917	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5918	}
5919	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5920	{
5921	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5922	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5923	AssertRCReturn(rc, rc);
5924	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5925	if ( !fBlockInt
5926	&& !fBlockSti
5927	&& !fBlockMovSS)
5928	{
5929	uint8_t u8Interrupt;
5930	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5931	if (RT_SUCCESS(rc))
5932	{
5933	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5934	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5935	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5936	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5937	}
5938	else
5939	{
5940	/** @todo Does this actually happen? If not turn it into an assertion. */
5941	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5942	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5943	rc = VINF_SUCCESS;
5944	}
5945	}
5946	else
5947	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5948	}
5949
5950	/*
5951	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5952	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5953	*/
5954	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5955	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5956	int rc2 = VINF_SUCCESS;
5957	if ( fBlockSti
5958	\|\| fBlockMovSS)
5959	{
5960	if (!DBGFIsStepping(pVCpu))
5961	{
5962	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5963	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5964	{
5965	/*
5966	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5967	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5968	*/
5969	rc2 = VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5970	}
5971	}
5972	else
5973	{
5974	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5975	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5976	uIntrState = 0;
5977	}
5978	}
5979
5980	/*
5981	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5982	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5983	*/
5984	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5985	AssertRC(rc2);
5986
5987	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5988	return rc;
5989	}
5990
5991
5992	/**
5993	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5994	*
5995	* @param pVCpu Pointer to the VMCPU.
5996	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5997	* out-of-sync. Make sure to update the required fields
5998	* before using them.
5999	*/
6000	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6001	{
6002	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6003	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6004	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6005	}
6006
6007
6008	/**
6009	* Injects a double-fault (#DF) exception into the VM.
6010	*
6011	* @returns VBox status code (informational status code included).
6012	* @param pVCpu Pointer to the VMCPU.
6013	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6014	* out-of-sync. Make sure to update the required fields
6015	* before using them.
6016	*/
6017	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6018	{
6019	/* Inject the double-fault. */
6020	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6021	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6022	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6023	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6024	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6025	}
6026
6027
6028	/**
6029	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6030	*
6031	* @param pVCpu Pointer to the VMCPU.
6032	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6033	* out-of-sync. Make sure to update the required fields
6034	* before using them.
6035	*/
6036	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6037	{
6038	/* Inject the debug-exception. */
6039	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6040	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6041	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6042	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6043	}
6044
6045
6046	/**
6047	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6048	*
6049	* @param pVCpu Pointer to the VMCPU.
6050	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6051	* out-of-sync. Make sure to update the required fields
6052	* before using them.
6053	* @param cbInstr The value of RIP that is to be pushed on the guest
6054	* stack.
6055	*/
6056	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6057	{
6058	/* Inject the overflow exception. */
6059	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6060	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6061	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6062	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6063	}
6064
6065
6066	/**
6067	* Injects a general-protection (#GP) fault into the VM.
6068	*
6069	* @returns VBox status code (informational status code included).
6070	* @param pVCpu Pointer to the VMCPU.
6071	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6072	* out-of-sync. Make sure to update the required fields
6073	* before using them.
6074	* @param u32ErrorCode The error code associated with the #GP.
6075	*/
6076	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6077	uint32_t *puIntrState)
6078	{
6079	/* Inject the general-protection fault. */
6080	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6081	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6082	if (fErrorCodeValid)
6083	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6084	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6085	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6086	}
6087
6088
6089	/**
6090	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6091	*
6092	* @param pVCpu Pointer to the VMCPU.
6093	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6094	* out-of-sync. Make sure to update the required fields
6095	* before using them.
6096	* @param uVector The software interrupt vector number.
6097	* @param cbInstr The value of RIP that is to be pushed on the guest
6098	* stack.
6099	*/
6100	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6101	{
6102	/* Inject the INTn. */
6103	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6104	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6105	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6106	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6107	}
6108
6109
6110	/**
6111	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6112	* stack.
6113	*
6114	* @returns VBox status code (information status code included).
6115	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6116	* @param pVM Pointer to the VM.
6117	* @param pMixedCtx Pointer to the guest-CPU context.
6118	* @param uValue The value to push to the guest stack.
6119	*/
6120	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6121	{
6122	/*
6123	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6124	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6125	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6126	*/
6127	if (pMixedCtx->sp == 1)
6128	return VINF_EM_RESET;
6129	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6130	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6131	AssertRCReturn(rc, rc);
6132	return rc;
6133	}
6134
6135
6136	/**
6137	* Injects an event into the guest upon VM-entry by updating the relevant fields
6138	* in the VM-entry area in the VMCS.
6139	*
6140	* @returns VBox status code (informational error codes included).
6141	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6142	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6143	*
6144	* @param pVCpu Pointer to the VMCPU.
6145	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6146	* out-of-sync. Make sure to update the required fields
6147	* before using them.
6148	* @param u64IntrInfo The VM-entry interruption-information field.
6149	* @param cbInstr The VM-entry instruction length in bytes (for software
6150	* interrupts, exceptions and privileged software
6151	* exceptions).
6152	* @param u32ErrCode The VM-entry exception error code.
6153	* @param puIntrState Pointer to the current guest interruptibility-state.
6154	* This interruptibility-state will be updated if
6155	* necessary. This cannot not be NULL.
6156	*
6157	* @remarks No-long-jump zone!!!
6158	* @remarks Requires CR0!
6159	*/
6160	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6161	uint32_t u32ErrCode, uint32_t *puIntrState)
6162	{
6163	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6164	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6165	Assert(puIntrState);
6166	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6167
6168	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6169	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6170
6171	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6172	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6173	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6174
6175	/* We require CR0 to check if the guest is in real-mode. */
6176	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6177	AssertRCReturn(rc, rc);
6178
6179	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6180
6181	/*
6182	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6183	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6184	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6185	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6186	*/
6187	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6188	{
6189	PVM pVM = pVCpu->CTX_SUFF(pVM);
6190	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6191	{
6192	Assert(PDMVmmDevHeapIsEnabled(pVM));
6193	Assert(pVM->hm.s.vmx.pRealModeTSS);
6194
6195	/* Save the required guest state bits from the VMCS. */
6196	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6197	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6198	AssertRCReturn(rc, rc);
6199
6200	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6201	const size_t cbIdtEntry = 4;
6202	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6203	{
6204	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6205	if (uVector == X86_XCPT_DF)
6206	return VINF_EM_RESET;
6207	else if (uVector == X86_XCPT_GP)
6208	{
6209	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6210	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6211	}
6212
6213	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6214	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6215	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6216	}
6217
6218	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6219	uint16_t uGuestIp = pMixedCtx->ip;
6220	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6221	{
6222	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6223	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6224	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6225	}
6226	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6227	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6228
6229	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6230	uint16_t offIdtEntry = 0;
6231	RTSEL selIdtEntry = 0;
6232	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6233	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6234	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6235	AssertRCReturn(rc, rc);
6236
6237	/* Construct the stack frame for the interrupt/exception handler. */
6238	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6239	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6240	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6241	AssertRCReturn(rc, rc);
6242
6243	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6244	if (rc == VINF_SUCCESS)
6245	{
6246	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6247	pMixedCtx->rip = offIdtEntry;
6248	pMixedCtx->cs.Sel = selIdtEntry;
6249	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6250	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6251	\| HM_CHANGED_GUEST_RIP
6252	\| HM_CHANGED_GUEST_RFLAGS
6253	\| HM_CHANGED_GUEST_RSP;
6254
6255	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6256	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6257	{
6258	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6259	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6260	Log(("Clearing inhibition due to STI.\n"));
6261	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6262	}
6263	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6264	}
6265	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6266	return rc;
6267	}
6268	else
6269	{
6270	/*
6271	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6272	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6273	*/
6274	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6275	}
6276	}
6277
6278	/* Validate. */
6279	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6280	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6281	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6282
6283	/* Inject. */
6284	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6285	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6286	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6287	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6288	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6289	AssertRCReturn(rc, rc);
6290	return rc;
6291	}
6292
6293
6294	/**
6295	* Enters the VT-x session.
6296	*
6297	* @returns VBox status code.
6298	* @param pVM Pointer to the VM.
6299	* @param pVCpu Pointer to the VMCPU.
6300	* @param pCpu Pointer to the CPU info struct.
6301	*/
6302	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6303	{
6304	AssertPtr(pVM);
6305	AssertPtr(pVCpu);
6306	Assert(pVM->hm.s.vmx.fSupported);
6307	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6308	NOREF(pCpu);
6309
6310	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6311
6312	/* Make sure we're in VMX root mode. */
6313	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6314	if (!(u32HostCR4 & X86_CR4_VMXE))
6315	{
6316	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6317	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6318	}
6319
6320	/* Load the active VMCS as the current one. */
6321	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6322	if (RT_FAILURE(rc))
6323	return rc;
6324
6325	/** @todo this will change with preemption hooks where can can VMRESUME as long
6326	* as we're no preempted. */
6327	pVCpu->hm.s.fResumeVM = false;
6328	return VINF_SUCCESS;
6329	}
6330
6331
6332	/**
6333	* Leaves the VT-x session.
6334	*
6335	* @returns VBox status code.
6336	* @param pVM Pointer to the VM.
6337	* @param pVCpu Pointer to the VMCPU.
6338	* @param pCtx Pointer to the guest-CPU context.
6339	*/
6340	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6341	{
6342	AssertPtr(pVCpu);
6343	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6344	NOREF(pVM);
6345	NOREF(pCtx);
6346
6347	/** @todo this will change with preemption hooks where we only VMCLEAR when
6348	* we are actually going to be preempted, not all the time like we
6349	* currently do. */
6350	/*
6351	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6352	* and mark the VMCS launch-state as "clear".
6353	*/
6354	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6355	return rc;
6356	}
6357
6358
6359	/**
6360	* Saves the host state in the VMCS host-state.
6361	* Sets up the VM-exit MSR-load area.
6362	*
6363	* The CPU state will be loaded from these fields on every successful VM-exit.
6364	*
6365	* @returns VBox status code.
6366	* @param pVM Pointer to the VM.
6367	* @param pVCpu Pointer to the VMCPU.
6368	*
6369	* @remarks No-long-jump zone!!!
6370	*/
6371	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6372	{
6373	AssertPtr(pVM);
6374	AssertPtr(pVCpu);
6375	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6376
6377	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6378
6379	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6380	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6381	return VINF_SUCCESS;
6382
6383	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6384	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6385
6386	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6387	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6388
6389	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6390	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6391
6392	/* Reset the host-state-changed flag. */
6393	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6394	return rc;
6395	}
6396
6397
6398	/**
6399	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6400	* loaded from these fields on every successful VM-entry.
6401	*
6402	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6403	* Sets up the VM-entry controls.
6404	* Sets up the appropriate VMX non-root function to execute guest code based on
6405	* the guest CPU mode.
6406	*
6407	* @returns VBox status code.
6408	* @param pVM Pointer to the VM.
6409	* @param pVCpu Pointer to the VMCPU.
6410	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6411	* out-of-sync. Make sure to update the required fields
6412	* before using them.
6413	*
6414	* @remarks No-long-jump zone!!!
6415	*/
6416	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6417	{
6418	AssertPtr(pVM);
6419	AssertPtr(pVCpu);
6420	AssertPtr(pMixedCtx);
6421	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6422
6423	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6424
6425	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6426
6427	/* Determine real-on-v86 mode. */
6428	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6429	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6430	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6431	{
6432	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6433	}
6434
6435	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6436	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6437
6438	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6439	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6440
6441	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6442	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6443
6444	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6445	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6446
6447	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6448	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6449
6450	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6451	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6452
6453	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6454	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6455
6456	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6457	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6458
6459	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6460	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6461
6462	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6463	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6464
6465	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6466	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6467	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6468
6469	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6470	return rc;
6471	}
6472
6473
6474	/**
6475	* Does the preparations before executing guest code in VT-x.
6476	*
6477	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6478	* recompiler. We must be cautious what we do here regarding committing
6479	* guest-state information into the the VMCS assuming we assuredly execute the
6480	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6481	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6482	* that the recompiler can (and should) use them when it resumes guest
6483	* execution. Otherwise such operations must be done when we can no longer
6484	* exit to ring-3.
6485	*
6486	* @returns VBox status code (informational status codes included).
6487	* @retval VINF_SUCCESS if we can proceed with running the guest.
6488	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6489	* into the guest.
6490	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6491	*
6492	* @param pVM Pointer to the VM.
6493	* @param pVCpu Pointer to the VMCPU.
6494	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6495	* out-of-sync. Make sure to update the required fields
6496	* before using them.
6497	* @param pVmxTransient Pointer to the VMX transient structure.
6498	*
6499	* @remarks Called with preemption disabled.
6500	*/
6501	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6502	{
6503	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6504
6505	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6506	PGMRZDynMapFlushAutoSet(pVCpu);
6507	#endif
6508
6509	/* Check force flag actions that might require us to go back to ring-3. */
6510	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6511	if (rc != VINF_SUCCESS)
6512	return rc;
6513
6514	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6515	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6516	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6517	{
6518	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6519	RTGCPHYS GCPhysApicBase;
6520	GCPhysApicBase = pMixedCtx->msrApicBase;
6521	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6522
6523	/* Unalias any existing mapping. */
6524	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6525	AssertRCReturn(rc, rc);
6526
6527	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6528	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6529	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6530	AssertRCReturn(rc, rc);
6531
6532	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6533	}
6534
6535	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6536	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6537	pVmxTransient->uEFlags = ASMIntDisableFlags();
6538	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6539	{
6540	ASMSetFlags(pVmxTransient->uEFlags);
6541	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6542	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6543	return VINF_EM_RAW_INTERRUPT;
6544	}
6545	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6546	#endif
6547
6548	/*
6549	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6550	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6551	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6552	*/
6553	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6554	AssertRCReturn(rc, rc);
6555	return rc;
6556	}
6557
6558
6559	/**
6560	* Prepares to run guest code in VT-x and we've committed to doing so. This
6561	* means there is no backing out to ring-3 or anywhere else at this
6562	* point.
6563	*
6564	* @param pVM Pointer to the VM.
6565	* @param pVCpu Pointer to the VMCPU.
6566	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6567	* out-of-sync. Make sure to update the required fields
6568	* before using them.
6569	* @param pVmxTransient Pointer to the VMX transient structure.
6570	*
6571	* @remarks Called with preemption disabled.
6572	* @remarks No-long-jump zone!!!
6573	*/
6574	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6575	{
6576	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6577	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6578
6579	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6580	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6581	pVmxTransient->uEFlags = ASMIntDisableFlags();
6582	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6583	#endif
6584
6585	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6586	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6587	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6588	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6589	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6590	#endif
6591	int rc = VINF_SUCCESS;
6592	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6593	{
6594	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6595	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6596	}
6597	else if (pVCpu->hm.s.fContextUseFlags)
6598	{
6599	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6600	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6601	}
6602	AssertRC(rc);
6603	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6604
6605	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6606	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6607	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6608
6609	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6610	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6611
6612	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6613	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6614	{
6615	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6616	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6617	}
6618
6619	/*
6620	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6621	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6622	*/
6623	if (pVM->hm.s.fTPRPatchingActive)
6624	{
6625	Assert(!CPUMIsGuestInLongMode(pVCpu));
6626
6627	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6628	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6629	AssertRC(rc);
6630
6631	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6632	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6633	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6634	see hmR0VmxLoadGuestApicState(). */
6635	}
6636
6637	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6638	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6639	to start executing. */
6640	}
6641
6642
6643	/**
6644	* Performs some essential restoration of state after running guest code in
6645	* VT-x.
6646	*
6647	* @param pVM Pointer to the VM.
6648	* @param pVCpu Pointer to the VMCPU.
6649	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6650	* out-of-sync. Make sure to update the required fields
6651	* before using them.
6652	* @param pVmxTransient Pointer to the VMX transient structure.
6653	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6654	*
6655	* @remarks Called with interrupts disabled.
6656	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6657	* unconditionally when it is safe to do so.
6658	*/
6659	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6660	{
6661	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6662	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6663
6664	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6665	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6666	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6667	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6668	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6669
6670	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6671	{
6672	/** @todo Find a way to fix hardcoding a guestimate. */
6673	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6674	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6675	}
6676
6677	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6678	Assert(!(ASMGetFlags() & X86_EFL_IF));
6679	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6680
6681	/* Restore the effects of TPR patching if any. */
6682	if (pVM->hm.s.fTPRPatchingActive)
6683	{
6684	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6685	AssertRC(rc);
6686	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6687	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6688	}
6689
6690	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6691	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6692
6693	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6694	uint32_t uExitReason;
6695	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6696	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6697	AssertRC(rc);
6698	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6699	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6700
6701	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6702	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6703
6704	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6705	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6706	{
6707	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6708	return;
6709	}
6710
6711	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6712	{
6713	/* Update the guest interruptibility-state from the VMCS. */
6714	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6715	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6716	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6717	AssertRC(rc);
6718	#endif
6719	/*
6720	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6721	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6722	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6723	*/
6724	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6725	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6726	{
6727	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6728	AssertRC(rc);
6729	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6730	}
6731	}
6732	}
6733
6734
6735	/**
6736	* Runs the guest code using VT-x.
6737	*
6738	* @returns VBox status code.
6739	* @param pVM Pointer to the VM.
6740	* @param pVCpu Pointer to the VMCPU.
6741	* @param pCtx Pointer to the guest-CPU context.
6742	*
6743	* @remarks Called with preemption disabled.
6744	*/
6745	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6746	{
6747	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6748	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6749
6750	VMXTRANSIENT VmxTransient;
6751	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6752	int rc = VERR_INTERNAL_ERROR_5;
6753	uint32_t cLoops = 0;
6754	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6755
6756	for (;; cLoops++)
6757	{
6758	Assert(!HMR0SuspendPending());
6759	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6760	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6761	(unsigned)RTMpCpuId(), cLoops));
6762
6763	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6764	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6765	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6766	if (rc != VINF_SUCCESS)
6767	break;
6768
6769	/*
6770	* No longjmps to ring-3 from this point on!!!
6771	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6772	* This also disables flushing of the R0-logger instance (if any).
6773	*/
6774	VMMRZCallRing3Disable(pVCpu);
6775	VMMRZCallRing3RemoveNotification(pVCpu);
6776	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6777
6778	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6779	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6780
6781	/*
6782	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6783	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6784	*/
6785	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6786	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6787	{
6788	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6789	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6790	return rc;
6791	}
6792
6793	/* Handle the VM-exit. */
6794	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6795	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6796	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6797	#ifdef HM_PROFILE_EXIT_DISPATCH
6798	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6799	#endif
6800	#ifdef HMVMX_USE_FUNCTION_TABLE
6801	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6802	#else
6803	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6804	#endif
6805	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6806	if (rc != VINF_SUCCESS)
6807	break;
6808	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6809	{
6810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6811	rc = VINF_EM_RAW_INTERRUPT;
6812	break;
6813	}
6814	}
6815
6816	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6817	if (rc == VERR_EM_INTERPRETER)
6818	rc = VINF_EM_RAW_EMULATE_INSTR;
6819	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6820	return rc;
6821	}
6822
6823
6824	#ifndef HMVMX_USE_FUNCTION_TABLE
6825	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6826	{
6827	int rc;
6828	switch (rcReason)
6829	{
6830	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6856	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6857	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6862	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6863
6864	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6865	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6866	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6867	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6868	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6869	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6870	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6871	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6872	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6873
6874	case VMX_EXIT_VMCALL:
6875	case VMX_EXIT_VMCLEAR:
6876	case VMX_EXIT_VMLAUNCH:
6877	case VMX_EXIT_VMPTRLD:
6878	case VMX_EXIT_VMPTRST:
6879	case VMX_EXIT_VMREAD:
6880	case VMX_EXIT_VMRESUME:
6881	case VMX_EXIT_VMWRITE:
6882	case VMX_EXIT_VMXOFF:
6883	case VMX_EXIT_VMXON:
6884	case VMX_EXIT_INVEPT:
6885	case VMX_EXIT_INVVPID:
6886	case VMX_EXIT_VMFUNC:
6887	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6888	break;
6889	default:
6890	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6891	break;
6892	}
6893	return rc;
6894	}
6895	#endif
6896
6897
6898	/** Profiling macro. */
6899	#ifdef HM_PROFILE_EXIT_DISPATCH
6900	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6901	#else
6902	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6903	#endif
6904
6905
6906	#ifdef DEBUG
6907	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6908	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6909	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6910	# define VMX_ASSERT_PREEMPT_CPUID() \
6911	do \
6912	{ \
6913	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6914	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6915	} while (0)
6916
6917	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6918	do { \
6919	AssertPtr(pVCpu); \
6920	AssertPtr(pMixedCtx); \
6921	AssertPtr(pVmxTransient); \
6922	Assert(pVmxTransient->fVMEntryFailed == false); \
6923	Assert(ASMIntAreEnabled()); \
6924	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6925	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6926	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6927	(unsigned)pVCpu->idCpu)); \
6928	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6929	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6930	VMX_ASSERT_PREEMPT_CPUID(); \
6931	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6932	} while (0)
6933	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6934	do { \
6935	LogFunc(("\n")); \
6936	} while(0)
6937	#else /* Release builds */
6938	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6939	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6940	#endif
6941
6942
6943	/**
6944	* Advances the guest RIP after reading it from the VMCS.
6945	*
6946	* @returns VBox status code.
6947	* @param pVCpu Pointer to the VMCPU.
6948	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6949	* out-of-sync. Make sure to update the required fields
6950	* before using them.
6951	* @param pVmxTransient Pointer to the VMX transient structure.
6952	*
6953	* @remarks No-long-jump zone!!!
6954	*/
6955	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6956	{
6957	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6958	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6959	AssertRCReturn(rc, rc);
6960
6961	pMixedCtx->rip += pVmxTransient->cbInstr;
6962	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6963	return rc;
6964	}
6965
6966
6967	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6968	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6969	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6970	/**
6971	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6972	*/
6973	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6974	{
6975	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6976	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6977	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6978	Assert(ASMIntAreEnabled());
6979	return VINF_SUCCESS;
6980	#else
6981	return VINF_EM_RAW_INTERRUPT;
6982	#endif
6983	}
6984
6985
6986	/**
6987	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6988	*/
6989	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6990	{
6991	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6992	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6993
6994	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6995	AssertRCReturn(rc, rc);
6996
6997	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6998	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6999	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7000
7001	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7002	{
7003	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7004	return VINF_EM_RAW_INTERRUPT;
7005	}
7006
7007	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7008	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7009	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7010	{
7011	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7012	return VINF_SUCCESS;
7013	}
7014	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7015	{
7016	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7017	return rc;
7018	}
7019
7020	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7021	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7022	switch (uIntrType)
7023	{
7024	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7025	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7026	/* no break */
7027	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
7028	{
7029	switch (uVector)
7030	{
7031	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7032	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7033	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7034	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7035	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7036	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7037	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7038	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7039	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7040	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7041	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7042	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7043	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7044	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7045	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7046	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7047	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7048	#endif
7049	default:
7050	{
7051	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7052	AssertRCReturn(rc, rc);
7053
7054	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7055	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7056	{
7057	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7058	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7059	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7060	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7061	AssertRCReturn(rc, rc);
7062	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7063	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7064	0 /* GCPtrFaultAddress */);
7065	AssertRCReturn(rc, rc);
7066	}
7067	else
7068	{
7069	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7070	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7071	}
7072	break;
7073	}
7074	}
7075	break;
7076	}
7077
7078	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7079	default:
7080	{
7081	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7082	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7083	break;
7084	}
7085	}
7086	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7087	return rc;
7088	}
7089
7090
7091	/**
7092	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7093	*/
7094	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7095	{
7096	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7097
7098	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7099	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7100	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7101	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7102	AssertRCReturn(rc, rc);
7103
7104	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7105	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7106	return VINF_SUCCESS;
7107	}
7108
7109
7110	/**
7111	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7112	*/
7113	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7114	{
7115	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7116	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7117	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7118	}
7119
7120
7121	/**
7122	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7123	*/
7124	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7125	{
7126	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7127	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7128	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7129	}
7130
7131
7132	/**
7133	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7134	*/
7135	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7136	{
7137	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7138	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7139	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7140	}
7141
7142
7143	/**
7144	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7145	*/
7146	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7147	{
7148	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7149	PVM pVM = pVCpu->CTX_SUFF(pVM);
7150	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7151	if (RT_LIKELY(rc == VINF_SUCCESS))
7152	{
7153	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7154	Assert(pVmxTransient->cbInstr == 2);
7155	}
7156	else
7157	{
7158	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7159	rc = VERR_EM_INTERPRETER;
7160	}
7161	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7162	return rc;
7163	}
7164
7165
7166	/**
7167	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7168	*/
7169	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7170	{
7171	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7172	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7173	AssertRCReturn(rc, rc);
7174
7175	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7176	return VINF_EM_RAW_EMULATE_INSTR;
7177
7178	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7179	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7180	}
7181
7182
7183	/**
7184	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7185	*/
7186	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7187	{
7188	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7189	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7190	AssertRCReturn(rc, rc);
7191
7192	PVM pVM = pVCpu->CTX_SUFF(pVM);
7193	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7194	if (RT_LIKELY(rc == VINF_SUCCESS))
7195	{
7196	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7197	Assert(pVmxTransient->cbInstr == 2);
7198	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7199	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7200	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7201	}
7202	else
7203	{
7204	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7205	rc = VERR_EM_INTERPRETER;
7206	}
7207	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7208	return rc;
7209	}
7210
7211
7212	/**
7213	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7214	*/
7215	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7216	{
7217	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7218	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7219	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7220	AssertRCReturn(rc, rc);
7221
7222	PVM pVM = pVCpu->CTX_SUFF(pVM);
7223	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7224	if (RT_LIKELY(rc == VINF_SUCCESS))
7225	{
7226	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7227	Assert(pVmxTransient->cbInstr == 3);
7228	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7229	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7230	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7231	}
7232	else
7233	{
7234	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7235	rc = VERR_EM_INTERPRETER;
7236	}
7237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7238	return rc;
7239	}
7240
7241
7242	/**
7243	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7244	*/
7245	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7246	{
7247	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7248	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7249	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7250	AssertRCReturn(rc, rc);
7251
7252	PVM pVM = pVCpu->CTX_SUFF(pVM);
7253	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7254	if (RT_LIKELY(rc == VINF_SUCCESS))
7255	{
7256	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7257	Assert(pVmxTransient->cbInstr == 2);
7258	}
7259	else
7260	{
7261	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7262	rc = VERR_EM_INTERPRETER;
7263	}
7264	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7265	return rc;
7266	}
7267
7268
7269	/**
7270	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7271	*/
7272	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7273	{
7274	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7275	PVM pVM = pVCpu->CTX_SUFF(pVM);
7276	Assert(!pVM->hm.s.fNestedPaging);
7277
7278	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7279	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7280	AssertRCReturn(rc, rc);
7281
7282	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7283	rc = VBOXSTRICTRC_VAL(rc2);
7284	if (RT_LIKELY(rc == VINF_SUCCESS))
7285	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7286	else
7287	{
7288	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7289	pVmxTransient->uExitQualification, rc));
7290	}
7291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7292	return rc;
7293	}
7294
7295
7296	/**
7297	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7298	*/
7299	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7300	{
7301	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7302	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7303	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7304	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7305	AssertRCReturn(rc, rc);
7306
7307	PVM pVM = pVCpu->CTX_SUFF(pVM);
7308	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7309	if (RT_LIKELY(rc == VINF_SUCCESS))
7310	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7311	else
7312	{
7313	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7314	rc = VERR_EM_INTERPRETER;
7315	}
7316	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7317	return rc;
7318	}
7319
7320
7321	/**
7322	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7323	*/
7324	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7325	{
7326	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7327	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7328	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7329	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7330	AssertRCReturn(rc, rc);
7331
7332	PVM pVM = pVCpu->CTX_SUFF(pVM);
7333	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7334	rc = VBOXSTRICTRC_VAL(rc2);
7335	if (RT_LIKELY( rc == VINF_SUCCESS
7336	\|\| rc == VINF_EM_HALT))
7337	{
7338	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7339	AssertRCReturn(rc3, rc3);
7340
7341	if ( rc == VINF_EM_HALT
7342	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7343	{
7344	rc = VINF_SUCCESS;
7345	}
7346	}
7347	else
7348	{
7349	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7350	rc = VERR_EM_INTERPRETER;
7351	}
7352	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7353	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7354	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7355	return rc;
7356	}
7357
7358
7359	/**
7360	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7361	*/
7362	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7363	{
7364	/*
7365	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7366	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7367	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7368	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7369	*/
7370	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7371	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7372	}
7373
7374
7375	/**
7376	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7377	*/
7378	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7379	{
7380	/*
7381	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7382	* root operation. If we get there there is something funny going on.
7383	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7384	*/
7385	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7386	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7387	}
7388
7389
7390	/**
7391	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7392	*/
7393	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7394	{
7395	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7396	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7397	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7398	}
7399
7400
7401	/**
7402	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7403	*/
7404	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7405	{
7406	/*
7407	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7408	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7409	* See Intel spec. 25.3 "Other Causes of VM-exits".
7410	*/
7411	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7412	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7413	}
7414
7415
7416	/**
7417	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7418	* VM-exit.
7419	*/
7420	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7421	{
7422	/*
7423	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7424	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7425	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7426	*/
7427	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7428	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7429	}
7430
7431
7432	/**
7433	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7434	* VM-exit.
7435	*/
7436	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7437	{
7438	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7439	return VINF_EM_RESET;
7440	}
7441
7442
7443	/**
7444	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7445	*/
7446	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7447	{
7448	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7449	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7450	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7451	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7452	AssertRCReturn(rc, rc);
7453
7454	pMixedCtx->rip++;
7455	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7456	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7457	rc = VINF_SUCCESS;
7458	else
7459	rc = VINF_EM_HALT;
7460
7461	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7462	return rc;
7463	}
7464
7465
7466	/**
7467	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7468	*/
7469	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7470	{
7471	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7472	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7473	return VINF_SUCCESS;
7474	}
7475
7476
7477	/**
7478	* VM-exit handler for expiry of the VMX preemption timer.
7479	*/
7480	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7481	{
7482	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7483
7484	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7485	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7486	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7487
7488	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7489	PVM pVM = pVCpu->CTX_SUFF(pVM);
7490	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7491	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7492	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7493	}
7494
7495
7496	/**
7497	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7498	*/
7499	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7500	{
7501	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7502	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7503	/** @todo check if XSETBV is supported by the recompiler. */
7504	return VERR_EM_INTERPRETER;
7505	}
7506
7507
7508	/**
7509	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7510	*/
7511	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7512	{
7513	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7514	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7515	/** @todo implement EMInterpretInvpcid() */
7516	return VERR_EM_INTERPRETER;
7517	}
7518
7519
7520	/**
7521	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7522	* Error VM-exit.
7523	*/
7524	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7525	{
7526	uint32_t uIntrState;
7527	RTHCUINTREG uHCReg;
7528	uint64_t u64Val;
7529
7530	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7531	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7532	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7533	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7534	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7535	AssertRCReturn(rc, rc);
7536
7537	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7538	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7539	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7540	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7541
7542	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7543	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7544	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7545	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7546	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7547	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7548	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7549	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7550	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7551	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7552	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7553	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7554
7555	PVM pVM = pVCpu->CTX_SUFF(pVM);
7556	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7557
7558	return VERR_VMX_INVALID_GUEST_STATE;
7559	}
7560
7561
7562	/**
7563	* VM-exit handler for VM-entry failure due to an MSR-load
7564	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7565	*/
7566	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7567	{
7568	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7569	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7570	}
7571
7572
7573	/**
7574	* VM-exit handler for VM-entry failure due to a machine-check event
7575	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7576	*/
7577	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7578	{
7579	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7580	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7581	}
7582
7583
7584	/**
7585	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7586	* theory.
7587	*/
7588	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7589	{
7590	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7591	return VERR_VMX_UNDEFINED_EXIT_CODE;
7592	}
7593
7594
7595	/**
7596	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7597	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7598	* Conditional VM-exit.
7599	*/
7600	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7601	{
7602	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7603	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7604	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7605	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7606	return VERR_EM_INTERPRETER;
7607	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7608	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7609	}
7610
7611
7612	/**
7613	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7614	*/
7615	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7616	{
7617	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7618	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7619	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7620	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7621	return VERR_EM_INTERPRETER;
7622	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7623	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7624	}
7625
7626
7627	/**
7628	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7629	*/
7630	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7631	{
7632	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7633	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7634	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7635	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7636	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7637	AssertRCReturn(rc, rc);
7638
7639	PVM pVM = pVCpu->CTX_SUFF(pVM);
7640	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7641	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7642	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7643	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7644
7645	/* Update RIP and continue guest execution. */
7646	if (RT_LIKELY(rc == VINF_SUCCESS))
7647	{
7648	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7649	Assert(pVmxTransient->cbInstr == 2);
7650	}
7651	return rc;
7652	}
7653
7654
7655	/**
7656	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7657	*/
7658	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7659	{
7660	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7661	PVM pVM = pVCpu->CTX_SUFF(pVM);
7662	int rc = VINF_SUCCESS;
7663
7664	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7665	if ( pVM->hm.s.fTPRPatchingActive
7666	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7667	{
7668	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7669	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7670	{
7671	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7672	AssertRC(rc);
7673	}
7674
7675	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7676	Assert(pVmxTransient->cbInstr == 2);
7677	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7678	return VINF_SUCCESS;
7679	}
7680
7681	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7682	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7683	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7684	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7685	AssertRCReturn(rc, rc);
7686	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7687
7688	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7689	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7690	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7691
7692	/* Update guest-state and continue execution. */
7693	if (RT_LIKELY(rc == VINF_SUCCESS))
7694	{
7695	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7696
7697	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7698	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7699	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7700	{
7701	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7702	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7703	}
7704	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7705	{
7706	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7707	AssertRCReturn(rc, rc);
7708	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7709	}
7710
7711	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7712	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7713	{
7714	switch (pMixedCtx->ecx)
7715	{
7716	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7717	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7718	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7719	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7720	case MSR_K8_FS_BASE: /* no break */
7721	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7722	}
7723	}
7724	#ifdef VBOX_STRICT
7725	else
7726	{
7727	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7728	switch (pMixedCtx->ecx)
7729	{
7730	case MSR_IA32_SYSENTER_CS:
7731	case MSR_IA32_SYSENTER_EIP:
7732	case MSR_IA32_SYSENTER_ESP:
7733	case MSR_K8_FS_BASE:
7734	case MSR_K8_GS_BASE:
7735	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7736	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7737
7738	case MSR_K8_LSTAR:
7739	case MSR_K6_STAR:
7740	case MSR_K8_SF_MASK:
7741	case MSR_K8_TSC_AUX:
7742	{
7743	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7744	pMixedCtx->ecx));
7745	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7746	}
7747
7748	case MSR_K8_KERNEL_GS_BASE:
7749	{
7750	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7751	pMixedCtx->ecx));
7752	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7753	}
7754	}
7755	}
7756	#endif /* VBOX_STRICT */
7757	}
7758	return rc;
7759	}
7760
7761
7762	/**
7763	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7764	*/
7765	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7766	{
7767	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7768	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7769	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7770	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7771	return VERR_EM_INTERPRETER;
7772	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7773	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7774	}
7775
7776
7777	/**
7778	* VM-exit handler for when the TPR value is lowered below the specified
7779	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7780	*/
7781	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7782	{
7783	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7784	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7785
7786	/*
7787	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7788	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7789	* resume guest execution.
7790	*/
7791	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7793	return VINF_SUCCESS;
7794	}
7795
7796
7797	/**
7798	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7799	* VM-exit.
7800	*
7801	* @retval VINF_SUCCESS when guest execution can continue.
7802	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7803	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7804	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7805	* recompiler.
7806	*/
7807	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7808	{
7809	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7810	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7811	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7812	AssertRCReturn(rc, rc);
7813
7814	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7815	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7816	PVM pVM = pVCpu->CTX_SUFF(pVM);
7817	switch (uAccessType)
7818	{
7819	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7820	{
7821	#if 0
7822	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7823	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7824	#else
7825	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7826	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7827	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7828	#endif
7829	AssertRCReturn(rc, rc);
7830
7831	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7832	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7833	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7834	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7835
7836	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7837	{
7838	case 0: /* CR0 */
7839	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7840	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7841	break;
7842	case 2: /* C2 **/
7843	/* Nothing to do here, CR2 it's not part of the VMCS. */
7844	break;
7845	case 3: /* CR3 */
7846	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7847	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7848	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7849	break;
7850	case 4: /* CR4 */
7851	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7852	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7853	break;
7854	case 8: /* CR8 */
7855	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7856	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7857	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7858	break;
7859	default:
7860	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7861	break;
7862	}
7863
7864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7865	break;
7866	}
7867
7868	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7869	{
7870	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7871	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7872	AssertRCReturn(rc, rc);
7873	Assert( !pVM->hm.s.fNestedPaging
7874	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7875	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7876
7877	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7878	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7879	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7880
7881	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7882	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7883	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7884	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7885	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7886	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7887	break;
7888	}
7889
7890	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7891	{
7892	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7893	AssertRCReturn(rc, rc);
7894	rc = EMInterpretCLTS(pVM, pVCpu);
7895	AssertRCReturn(rc, rc);
7896	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7897	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7898	Log(("CRX CLTS write rc=%d\n", rc));
7899	break;
7900	}
7901
7902	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7903	{
7904	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7905	AssertRCReturn(rc, rc);
7906	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7907	if (RT_LIKELY(rc == VINF_SUCCESS))
7908	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7910	Log(("CRX LMSW write rc=%d\n", rc));
7911	break;
7912	}
7913
7914	default:
7915	{
7916	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7917	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7918	}
7919	}
7920
7921	/* Validate possible error codes. */
7922	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7923	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7924	if (RT_SUCCESS(rc))
7925	{
7926	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7927	AssertRCReturn(rc2, rc2);
7928	}
7929
7930	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7931	return rc;
7932	}
7933
7934
7935	/**
7936	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7937	* VM-exit.
7938	*/
7939	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7940	{
7941	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7942	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7943
7944	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7945	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7946	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7947	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7948	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7949	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7950	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7951	AssertRCReturn(rc, rc);
7952	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7953
7954	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7955	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7956	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7957	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7958	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7959	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7960	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7961
7962	/* I/O operation lookup arrays. */
7963	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7964	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7965
7966	const uint32_t cbSize = s_aIOSize[uIOWidth];
7967	const uint32_t cbInstr = pVmxTransient->cbInstr;
7968	PVM pVM = pVCpu->CTX_SUFF(pVM);
7969	if (fIOString)
7970	{
7971	/* INS/OUTS - I/O String instruction. */
7972	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7973	/** @todo for now manually disassemble later optimize by getting the fields from
7974	* the VMCS. */
7975	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7976	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7977	* segment prefix info. */
7978	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7979	if (RT_SUCCESS(rc))
7980	{
7981	if (fIOWrite)
7982	{
7983	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7984	(DISCPUMODE)pDis->uAddrMode, cbSize);
7985	rc = VBOXSTRICTRC_VAL(rc2);
7986	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7987	}
7988	else
7989	{
7990	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7991	(DISCPUMODE)pDis->uAddrMode, cbSize);
7992	rc = VBOXSTRICTRC_VAL(rc2);
7993	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7994	}
7995	}
7996	else
7997	{
7998	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7999	rc = VINF_EM_RAW_EMULATE_INSTR;
8000	}
8001	}
8002	else
8003	{
8004	/* IN/OUT - I/O instruction. */
8005	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8006	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8007	if (fIOWrite)
8008	{
8009	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8010	rc = VBOXSTRICTRC_VAL(rc2);
8011	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8012	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8013	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8014	}
8015	else
8016	{
8017	uint32_t u32Result = 0;
8018	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8019	rc = VBOXSTRICTRC_VAL(rc2);
8020	if (IOM_SUCCESS(rc))
8021	{
8022	/* Save result of I/O IN instr. in AL/AX/EAX. */
8023	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8024	}
8025	else if (rc == VINF_IOM_R3_IOPORT_READ)
8026	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8027	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8028	}
8029	}
8030
8031	if (IOM_SUCCESS(rc))
8032	{
8033	pMixedCtx->rip += cbInstr;
8034	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8035	if (RT_LIKELY(rc == VINF_SUCCESS))
8036	{
8037	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8038	AssertRCReturn(rc, rc);
8039
8040	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8041	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8042	{
8043	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8044	for (unsigned i = 0; i < 4; i++)
8045	{
8046	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8047	if ( ( uIOPort >= pMixedCtx->dr[i]
8048	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8049	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8050	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8051	{
8052	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8053	uint64_t uDR6 = ASMGetDR6();
8054
8055	/* Clear all breakpoint status flags and set the one we just hit. */
8056	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8057	uDR6 \|= (uint64_t)RT_BIT(i);
8058
8059	/*
8060	* Note: AMD64 Architecture Programmer's Manual 13.1:
8061	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8062	* be cleared by software after the contents have been read.
8063	*/
8064	ASMSetDR6(uDR6);
8065
8066	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8067	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8068
8069	/* Paranoia. */
8070	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8071	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8072	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8073
8074	/* Resync DR7 */
8075	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8076	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8077
8078	/* Set #DB to be injected into the VM and continue guest execution. */
8079	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8080	break;
8081	}
8082	}
8083	}
8084	}
8085	}
8086
8087	#ifdef DEBUG
8088	if (rc == VINF_IOM_R3_IOPORT_READ)
8089	Assert(!fIOWrite);
8090	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8091	Assert(fIOWrite);
8092	else
8093	{
8094	AssertMsg( RT_FAILURE(rc)
8095	\|\| rc == VINF_SUCCESS
8096	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8097	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8098	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8099	}
8100	#endif
8101
8102	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8103	return rc;
8104	}
8105
8106
8107	/**
8108	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8109	* VM-exit.
8110	*/
8111	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8112	{
8113	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8114
8115	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8116	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8117	AssertRCReturn(rc, rc);
8118	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8119	{
8120	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8121	AssertRCReturn(rc, rc);
8122	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8123	{
8124	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8125	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8126	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8127	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8128	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8129	{
8130	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8131	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8132
8133	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8134	Assert(!pVCpu->hm.s.Event.fPending);
8135	pVCpu->hm.s.Event.fPending = true;
8136	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8137	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8138	AssertRCReturn(rc, rc);
8139	if (fErrorCodeValid)
8140	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8141	else
8142	pVCpu->hm.s.Event.u32ErrCode = 0;
8143	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8144	&& uVector == X86_XCPT_PF)
8145	{
8146	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8147	}
8148	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8149	}
8150	}
8151	}
8152	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8153	* emulation. */
8154	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8155	return VERR_EM_INTERPRETER;
8156	}
8157
8158
8159	/**
8160	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8161	*/
8162	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8163	{
8164	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8165	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8166	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8167	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8168	AssertRCReturn(rc, rc);
8169	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8170	return VINF_EM_DBG_STOP;
8171	}
8172
8173
8174	/**
8175	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8176	*/
8177	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8178	{
8179	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8180
8181	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8182	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8183	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8184	return VINF_SUCCESS;
8185	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8186	return rc;
8187
8188	#if 0
8189	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8190	* just sync the whole thing. */
8191	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8192	#else
8193	/* Aggressive state sync. for now. */
8194	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8195	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8196	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8197	#endif
8198	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8199	AssertRCReturn(rc, rc);
8200
8201	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8202	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8203	switch (uAccessType)
8204	{
8205	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8206	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8207	{
8208	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8209	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8210	{
8211	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8212	}
8213
8214	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8215	GCPhys &= PAGE_BASE_GC_MASK;
8216	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8217	PVM pVM = pVCpu->CTX_SUFF(pVM);
8218	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8219	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8220
8221	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8222	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8223	CPUMCTX2CORE(pMixedCtx), GCPhys);
8224	rc = VBOXSTRICTRC_VAL(rc2);
8225	Log(("ApicAccess rc=%d\n", rc));
8226	if ( rc == VINF_SUCCESS
8227	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8228	\|\| rc == VERR_PAGE_NOT_PRESENT)
8229	{
8230	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8231	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8232	rc = VINF_SUCCESS;
8233	}
8234	break;
8235	}
8236
8237	default:
8238	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8239	rc = VINF_EM_RAW_EMULATE_INSTR;
8240	break;
8241	}
8242
8243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8244	return rc;
8245	}
8246
8247
8248	/**
8249	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8250	* VM-exit.
8251	*/
8252	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8253	{
8254	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8255
8256	/* We should -not- get this VM-exit if the guest is debugging. */
8257	if (CPUMIsGuestDebugStateActive(pVCpu))
8258	{
8259	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8260	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8261	}
8262
8263	int rc = VERR_INTERNAL_ERROR_5;
8264	if ( !DBGFIsStepping(pVCpu)
8265	&& !CPUMIsHyperDebugStateActive(pVCpu))
8266	{
8267	/* Don't intercept MOV DRx. */
8268	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8269	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8270	AssertRCReturn(rc, rc);
8271
8272	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8273	PVM pVM = pVCpu->CTX_SUFF(pVM);
8274	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8275	AssertRC(rc);
8276	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8277
8278	#ifdef VBOX_WITH_STATISTICS
8279	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8280	AssertRCReturn(rc, rc);
8281	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8283	else
8284	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8285	#endif
8286	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8287	return VINF_SUCCESS;
8288	}
8289
8290	/*
8291	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8292	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8293	*/
8294	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8295	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8296	AssertRCReturn(rc, rc);
8297
8298	PVM pVM = pVCpu->CTX_SUFF(pVM);
8299	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8300	{
8301	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8302	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8303	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8304	if (RT_SUCCESS(rc))
8305	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8306	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8307	}
8308	else
8309	{
8310	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8311	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8312	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8314	}
8315
8316	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8317	if (RT_SUCCESS(rc))
8318	{
8319	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8320	AssertRCReturn(rc2, rc2);
8321	}
8322	return rc;
8323	}
8324
8325
8326	/**
8327	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8328	* Conditional VM-exit.
8329	*/
8330	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8331	{
8332	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8333	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8334
8335	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8336	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8337	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8338	return VINF_SUCCESS;
8339	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8340	return rc;
8341
8342	RTGCPHYS GCPhys = 0;
8343	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8344
8345	#if 0
8346	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8347	#else
8348	/* Aggressive state sync. for now. */
8349	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8350	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8351	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8352	#endif
8353	AssertRCReturn(rc, rc);
8354
8355	/*
8356	* If we succeed, resume guest execution.
8357	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8358	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8359	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8360	* weird case. See @bugref{6043}.
8361	*/
8362	PVM pVM = pVCpu->CTX_SUFF(pVM);
8363	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8364	rc = VBOXSTRICTRC_VAL(rc2);
8365	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8366	if ( rc == VINF_SUCCESS
8367	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8368	\|\| rc == VERR_PAGE_NOT_PRESENT)
8369	{
8370	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8371	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8372	return VINF_SUCCESS;
8373	}
8374	return rc;
8375	}
8376
8377
8378	/**
8379	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8380	* VM-exit.
8381	*/
8382	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8383	{
8384	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8385	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8386
8387	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8388	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8389	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8390	return VINF_SUCCESS;
8391	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8392	return rc;
8393
8394	RTGCPHYS GCPhys = 0;
8395	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8396	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8397	#if 0
8398	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8399	#else
8400	/* Aggressive state sync. for now. */
8401	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8402	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8403	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8404	#endif
8405	AssertRCReturn(rc, rc);
8406
8407	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8408	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8409
8410	RTGCUINT uErrorCode = 0;
8411	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8412	uErrorCode \|= X86_TRAP_PF_ID;
8413	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8414	uErrorCode \|= X86_TRAP_PF_RW;
8415	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8416	uErrorCode \|= X86_TRAP_PF_P;
8417
8418	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8419
8420	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8421	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8422
8423	/* Handle the pagefault trap for the nested shadow table. */
8424	PVM pVM = pVCpu->CTX_SUFF(pVM);
8425	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8426	TRPMResetTrap(pVCpu);
8427
8428	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8429	if ( rc == VINF_SUCCESS
8430	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8431	\|\| rc == VERR_PAGE_NOT_PRESENT)
8432	{
8433	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8434	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8435	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8436	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8437	return VINF_SUCCESS;
8438	}
8439
8440	Log(("EPT return to ring-3 rc=%d\n"));
8441	return rc;
8442	}
8443
8444
8445	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8446	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8447	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8448	/**
8449	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8450	*/
8451	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8452	{
8453	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8455
8456	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8457	AssertRCReturn(rc, rc);
8458
8459	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8460	{
8461	/* Old-style FPU error reporting needs some extra work. */
8462	/** @todo don't fall back to the recompiler, but do it manually. */
8463	return VERR_EM_INTERPRETER;
8464	}
8465	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8466	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8467	return rc;
8468	}
8469
8470
8471	/**
8472	* VM-exit exception handler for #BP (Breakpoint exception).
8473	*/
8474	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8475	{
8476	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8477	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8478
8479	/** @todo Try optimize this by not saving the entire guest state unless
8480	* really needed. */
8481	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8482	AssertRCReturn(rc, rc);
8483
8484	PVM pVM = pVCpu->CTX_SUFF(pVM);
8485	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8486	if (rc == VINF_EM_RAW_GUEST_TRAP)
8487	{
8488	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8489	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8490	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8491	AssertRCReturn(rc, rc);
8492
8493	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8494	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8495	}
8496
8497	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8498	return rc;
8499	}
8500
8501
8502	/**
8503	* VM-exit exception handler for #DB (Debug exception).
8504	*/
8505	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8506	{
8507	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8508	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8509
8510	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8511	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8512	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8513	AssertRCReturn(rc, rc);
8514
8515	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8516	uint64_t uDR6 = X86_DR6_INIT_VAL;
8517	uDR6 \|= (pVmxTransient->uExitQualification
8518	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8519	PVM pVM = pVCpu->CTX_SUFF(pVM);
8520	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8521	if (rc == VINF_EM_RAW_GUEST_TRAP)
8522	{
8523	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8524	pMixedCtx->dr[6] = uDR6;
8525
8526	if (CPUMIsGuestDebugStateActive(pVCpu))
8527	ASMSetDR6(pMixedCtx->dr[6]);
8528
8529	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8530
8531	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8532	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8533
8534	/* Paranoia. */
8535	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8536	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8537	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8538
8539	/* Resync DR7. */
8540	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8541	AssertRCReturn(rc,rc);
8542
8543	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8544	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8545	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8546	AssertRCReturn(rc2, rc2);
8547	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8548	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8549	rc = VINF_SUCCESS;
8550	}
8551
8552	return rc;
8553	}
8554
8555
8556	/**
8557	* VM-exit exception handler for #NM (Device-not-available exception: floating
8558	* point exception).
8559	*/
8560	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8561	{
8562	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8563
8564	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8565	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8566	#endif
8567
8568	/* We require CR0 and EFER. EFER is always up-to-date. */
8569	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8570	AssertRCReturn(rc, rc);
8571
8572	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8573	PVM pVM = pVCpu->CTX_SUFF(pVM);
8574	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8575	if (rc == VINF_SUCCESS)
8576	{
8577	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8578	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8580	return VINF_SUCCESS;
8581	}
8582
8583	/* Forward #NM to the guest. */
8584	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8585	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8586	AssertRCReturn(rc, rc);
8587	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8588	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8589	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8590	return rc;
8591	}
8592
8593
8594	/**
8595	* VM-exit exception handler for #GP (General-protection exception).
8596	*
8597	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8598	*/
8599	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8600	{
8601	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8602	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8603
8604	int rc = VERR_INTERNAL_ERROR_5;
8605	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8606	{
8607	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8608	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8609	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8610	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8611	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8612	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8613	AssertRCReturn(rc, rc);
8614	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8615	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8616	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8617	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8618	return rc;
8619	#else
8620	/* We don't intercept #GP. */
8621	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8622	return VERR_VMX_UNEXPECTED_EXCEPTION;
8623	#endif
8624	}
8625
8626	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8627	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8628
8629	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8630	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8631	AssertRCReturn(rc, rc);
8632
8633	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8634	uint32_t cbOp = 0;
8635	PVM pVM = pVCpu->CTX_SUFF(pVM);
8636	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8637	if (RT_SUCCESS(rc))
8638	{
8639	rc = VINF_SUCCESS;
8640	Assert(cbOp == pDis->cbInstr);
8641	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8642	switch (pDis->pCurInstr->uOpcode)
8643	{
8644	case OP_CLI:
8645	pMixedCtx->eflags.Bits.u1IF = 0;
8646	pMixedCtx->rip += pDis->cbInstr;
8647	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8648	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8649	break;
8650
8651	case OP_STI:
8652	pMixedCtx->eflags.Bits.u1IF = 1;
8653	pMixedCtx->rip += pDis->cbInstr;
8654	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8655	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8656	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8657	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8658	break;
8659
8660	case OP_HLT:
8661	rc = VINF_EM_HALT;
8662	pMixedCtx->rip += pDis->cbInstr;
8663	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8664	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8665	break;
8666
8667	case OP_POPF:
8668	{
8669	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8670	uint32_t cbParm = 0;
8671	uint32_t uMask = 0;
8672	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8673	{
8674	cbParm = 4;
8675	uMask = 0xffffffff;
8676	}
8677	else
8678	{
8679	cbParm = 2;
8680	uMask = 0xffff;
8681	}
8682
8683	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8684	RTGCPTR GCPtrStack = 0;
8685	X86EFLAGS uEflags;
8686	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8687	&GCPtrStack);
8688	if (RT_SUCCESS(rc))
8689	{
8690	Assert(sizeof(uEflags.u32) >= cbParm);
8691	uEflags.u32 = 0;
8692	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8693	}
8694	if (RT_FAILURE(rc))
8695	{
8696	rc = VERR_EM_INTERPRETER;
8697	break;
8698	}
8699	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8700	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8701	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8702	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8703	pMixedCtx->eflags.Bits.u1RF = 0;
8704	pMixedCtx->esp += cbParm;
8705	pMixedCtx->esp &= uMask;
8706	pMixedCtx->rip += pDis->cbInstr;
8707	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8709	break;
8710	}
8711
8712	case OP_PUSHF:
8713	{
8714	uint32_t cbParm = 0;
8715	uint32_t uMask = 0;
8716	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8717	{
8718	cbParm = 4;
8719	uMask = 0xffffffff;
8720	}
8721	else
8722	{
8723	cbParm = 2;
8724	uMask = 0xffff;
8725	}
8726
8727	/* Get the stack pointer & push the contents of eflags onto the stack. */
8728	RTGCPTR GCPtrStack = 0;
8729	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8730	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8731	if (RT_FAILURE(rc))
8732	{
8733	rc = VERR_EM_INTERPRETER;
8734	break;
8735	}
8736	X86EFLAGS uEflags;
8737	uEflags = pMixedCtx->eflags;
8738	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8739	uEflags.Bits.u1RF = 0;
8740	uEflags.Bits.u1VM = 0;
8741
8742	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8743	if (RT_FAILURE(rc))
8744	{
8745	rc = VERR_EM_INTERPRETER;
8746	break;
8747	}
8748	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8749	pMixedCtx->esp -= cbParm;
8750	pMixedCtx->esp &= uMask;
8751	pMixedCtx->rip += pDis->cbInstr;
8752	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8753	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8754	break;
8755	}
8756
8757	case OP_IRET:
8758	{
8759	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8760	* instruction reference. */
8761	RTGCPTR GCPtrStack = 0;
8762	uint32_t uMask = 0xffff;
8763	uint16_t aIretFrame[3];
8764	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8765	{
8766	rc = VERR_EM_INTERPRETER;
8767	break;
8768	}
8769	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8770	&GCPtrStack);
8771	if (RT_SUCCESS(rc))
8772	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8773	if (RT_FAILURE(rc))
8774	{
8775	rc = VERR_EM_INTERPRETER;
8776	break;
8777	}
8778	pMixedCtx->eip = 0;
8779	pMixedCtx->ip = aIretFrame[0];
8780	pMixedCtx->cs.Sel = aIretFrame[1];
8781	pMixedCtx->cs.ValidSel = aIretFrame[1];
8782	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8783	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8784	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8785	pMixedCtx->sp += sizeof(aIretFrame);
8786	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8787	\| HM_CHANGED_GUEST_RFLAGS;
8788	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8790	break;
8791	}
8792
8793	case OP_INT:
8794	{
8795	uint16_t uVector = pDis->Param1.uValue & 0xff;
8796	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8797	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8798	break;
8799	}
8800
8801	case OP_INTO:
8802	{
8803	if (pMixedCtx->eflags.Bits.u1OF)
8804	{
8805	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8806	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8807	}
8808	break;
8809	}
8810
8811	default:
8812	{
8813	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8814	EMCODETYPE_SUPERVISOR);
8815	rc = VBOXSTRICTRC_VAL(rc2);
8816	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8817	Log(("#GP rc=%Rrc\n", rc));
8818	break;
8819	}
8820	}
8821	}
8822	else
8823	rc = VERR_EM_INTERPRETER;
8824
8825	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8826	("#GP Unexpected rc=%Rrc\n", rc));
8827	return rc;
8828	}
8829
8830
8831	/**
8832	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8833	* the exception reported in the VMX transient structure back into the VM.
8834	*
8835	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8836	* VMX transient structure to be up-to-date.
8837	*/
8838	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8839	{
8840	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8841
8842	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8843	hmR0VmxCheckExitDueToEventDelivery(). */
8844	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8845	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8846	AssertRCReturn(rc, rc);
8847	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8848	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8849	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8850	return VINF_SUCCESS;
8851	}
8852
8853
8854	/**
8855	* VM-exit exception handler for #PF (Page-fault exception).
8856	*/
8857	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8858	{
8859	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8860	PVM pVM = pVCpu->CTX_SUFF(pVM);
8861	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8862	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8863	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8864	AssertRCReturn(rc, rc);
8865
8866	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8867	if (pVM->hm.s.fNestedPaging)
8868	{
8869	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8870	{
8871	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8872	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8873	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8874	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8876	}
8877	else
8878	{
8879	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8880	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8881	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8882	Log(("Pending #DF due to vectoring #PF. NP\n"));
8883	}
8884	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8885	return rc;
8886	}
8887	#else
8888	Assert(!pVM->hm.s.fNestedPaging);
8889	#endif
8890
8891	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8892	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8893	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8894	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8895	AssertRCReturn(rc, rc);
8896	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8897	if ( pVM->hm.s.fTRPPatchingAllowed
8898	&& pVM->hm.s.pGuestPatchMem
8899	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8900	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8901	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8902	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8903	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8904	{
8905	RTGCPHYS GCPhys;
8906	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8907	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8908	if ( rc == VINF_SUCCESS
8909	&& GCPhys == GCPhysApicBase)
8910	{
8911	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8912	AssertRCReturn(rc, rc);
8913
8914	/* Only attempt to patch the instruction once. */
8915	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8916	if (!pPatch)
8917	return VINF_EM_HM_PATCH_TPR_INSTR;
8918	}
8919	}
8920	#endif
8921
8922	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8923	AssertRCReturn(rc, rc);
8924
8925	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8926	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8927
8928	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8929	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8930	(RTGCPTR)pVmxTransient->uExitQualification);
8931
8932	Log(("#PF: rc=%Rrc\n", rc));
8933	if (rc == VINF_SUCCESS)
8934	{
8935	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8936	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8937	* memory? We don't update the whole state here... */
8938	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8939	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8940	TRPMResetTrap(pVCpu);
8941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8942	return rc;
8943	}
8944	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8945	{
8946	if (!pVmxTransient->fVectoringPF)
8947	{
8948	/* It's a guest page fault and needs to be reflected to the guest. */
8949	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8950	TRPMResetTrap(pVCpu);
8951	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8952	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8953	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8954	0 /* cbInstr */, uGstErrorCode, pMixedCtx->cr2);
8955	}
8956	else
8957	{
8958	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8959	TRPMResetTrap(pVCpu);
8960	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8961	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8962	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8963	}
8964
8965	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8966	return VINF_SUCCESS;
8967	}
8968
8969	TRPMResetTrap(pVCpu);
8970	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8971	return rc;
8972	}
8973

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45655

Download in other formats: