HMVMXR0.cpp@ 45732

Last change on this file since 45732 was 45732, checked in by vboxsync, 12 years ago
VMM/HMVMXR0: Don't trash CR2 for VM-exits caused due to a #PF delivery when not intercepting #PFs (i.e. Nested Paging).
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 374.7 KB

Line
1	/* $Id: HMVMXR0.cpp 45732 2013-04-25 14:39:46Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#if defined(RT_ARCH_AMD64)
49	# define HMVMX_IS_64BIT_HOST_MODE() (true)
50	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
51	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
52	#else
53	# define HMVMX_IS_64BIT_HOST_MODE() (false)
54	#endif
55
56	/** Use the function table. */
57	#define HMVMX_USE_FUNCTION_TABLE
58
59	/** This bit indicates the segment selector is unusable in VT-x. */
60	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
61
62	/** Determine which tagged-TLB flush handler to use. */
63	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68	/** Updated-guest-state flags. */
69	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
70	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
71	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
72	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
73	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
74	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
75	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
76	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
77	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
78	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
79	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
80	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
81	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
82	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
83	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
84	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
85	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
86	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
87	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
88	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
89	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
90	\| HMVMX_UPDATED_GUEST_RSP \
91	\| HMVMX_UPDATED_GUEST_RFLAGS \
92	\| HMVMX_UPDATED_GUEST_CR0 \
93	\| HMVMX_UPDATED_GUEST_CR3 \
94	\| HMVMX_UPDATED_GUEST_CR4 \
95	\| HMVMX_UPDATED_GUEST_GDTR \
96	\| HMVMX_UPDATED_GUEST_IDTR \
97	\| HMVMX_UPDATED_GUEST_LDTR \
98	\| HMVMX_UPDATED_GUEST_TR \
99	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
100	\| HMVMX_UPDATED_GUEST_DEBUG \
101	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
102	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
103	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
104	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
105	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
106	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
107	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108	\| HMVMX_UPDATED_GUEST_APIC_STATE)
109
110	/**
111	* Flags to skip redundant reads of some common VMCS fields that are not part of
112	* the guest-CPU state but are in the transient structure.
113	*/
114	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
115	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
116	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
117	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
118	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
119	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
120
121	/**
122	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
123	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
124	* we have Nested Paging support.
125	*/
126	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
127	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
128	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
129	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
130	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
131	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
132	\| RT_BIT(X86_XCPT_XF))
133
134	/**
135	* Exception bitmap mask for all contributory exceptions.
136	*/
137	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
138	\| RT_BIT(X86_XCPT_DE))
139
140	/** Maximum VM-instruction error number. */
141	#define HMVMX_INSTR_ERROR_MAX 28
142
143
144	/*******************************************************************************
145	* Structures and Typedefs *
146	*******************************************************************************/
147	/**
148	* A state structure for holding miscellaneous information across
149	* VMX non-root operation and restored after the transition.
150	*/
151	typedef struct VMXTRANSIENT
152	{
153	/** The host's rflags/eflags. */
154	RTCCUINTREG uEFlags;
155	#if HC_ARCH_BITS == 32
156	uint32_t u32Alignment0;
157	#endif
158	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
159	uint64_t u64LStarMsr;
160	/** The guest's TPR value used for TPR shadowing. */
161	uint8_t u8GuestTpr;
162	/** Alignment. */
163	uint8_t abAlignment0[6];
164
165	/** The basic VM-exit reason. */
166	uint16_t uExitReason;
167	/** Alignment. */
168	uint16_t u16Alignment0;
169	/** The VM-exit interruption error code. */
170	uint32_t uExitIntrErrorCode;
171	/** The VM-exit exit qualification. */
172	RTGCUINTPTR uExitQualification;
173	#if GC_ARCH_BITS == 32
174	/** Alignment. */
175	uint32_t u32Alignment1;
176	#endif
177
178	/** The VM-exit interruption-information field. */
179	uint32_t uExitIntrInfo;
180	/** The VM-exit instruction-length field. */
181	uint32_t cbInstr;
182	/** Whether the VM-entry failed or not. */
183	bool fVMEntryFailed;
184	/** Alignment. */
185	uint8_t abAlignment1[5];
186
187	/** The VM-entry interruption-information field. */
188	uint32_t uEntryIntrInfo;
189	/** The VM-entry exception error code field. */
190	uint32_t uEntryXcptErrorCode;
191	/** The VM-entry instruction length field. */
192	uint32_t cbEntryInstr;
193
194	/** IDT-vectoring information field. */
195	uint32_t uIdtVectoringInfo;
196	/** IDT-vectoring error code. */
197	uint32_t uIdtVectoringErrorCode;
198
199	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
200	uint32_t fVmcsFieldsRead;
201	/** Whether TSC-offsetting should be setup before VM-entry. */
202	bool fUpdateTscOffsettingAndPreemptTimer;
203	/** Whether the VM-exit was caused by a page-fault during delivery of a
204	* contributary exception or a page-fault. */
205	bool fVectoringPF;
206	} VMXTRANSIENT, *PVMXTRANSIENT;
207	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
208	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
209	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
210
211
212	/**
213	* MSR-bitmap read permissions.
214	*/
215	typedef enum VMXMSREXITREAD
216	{
217	/** Reading this MSR causes a VM-exit. */
218	VMXMSREXIT_INTERCEPT_READ = 0xb,
219	/** Reading this MSR does not cause a VM-exit. */
220	VMXMSREXIT_PASSTHRU_READ
221	} VMXMSREXITREAD;
222
223	/**
224	* MSR-bitmap write permissions.
225	*/
226	typedef enum VMXMSREXITWRITE
227	{
228	/** Writing to this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
230	/** Writing to this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_WRITE
232	} VMXMSREXITWRITE;
233
234
235	/*******************************************************************************
236	* Internal Functions *
237	*******************************************************************************/
238	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
239	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
240	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
241	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
242	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
243	#endif
244	#ifndef HMVMX_USE_FUNCTION_TABLE
245	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
246	#endif
247
248	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292
293	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300
301
302	/*******************************************************************************
303	* Global Variables *
304	*******************************************************************************/
305	#ifdef HMVMX_USE_FUNCTION_TABLE
306	/**
307	* VM-exit handler.
308	*
309	* @returns VBox status code.
310	* @param pVCpu Pointer to the VMCPU.
311	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
312	* out-of-sync. Make sure to update the required
313	* fields before using them.
314	* @param pVmxTransient Pointer to the VMX-transient structure.
315	*/
316	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
317	/** Pointer to VM-exit handler. */
318	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
319
320	/**
321	* VMX_EXIT dispatch table.
322	*/
323	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
324	{
325	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
326	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
327	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
328	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
329	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
330	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
331	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
332	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
333	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
334	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
335	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
336	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
337	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
338	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
339	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
340	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
341	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
342	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
343	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
344	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
345	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
346	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
347	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
348	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
349	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
350	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
351	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
352	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
353	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
354	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
355	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
356	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
357	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
358	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
359	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
360	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
361	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
362	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
363	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
364	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
365	/* 40 UNDEFINED */ hmR0VmxExitPause,
366	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
367	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
368	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
369	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
370	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
371	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
372	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
373	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
374	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
375	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
376	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
377	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
378	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
379	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
380	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
381	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
382	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
383	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
384	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
385	};
386	#endif /* HMVMX_USE_FUNCTION_TABLE */
387
388	#ifdef VBOX_STRICT
389	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
390	{
391	/* 0 */ "(Not Used)",
392	/* 1 */ "VMCALL executed in VMX root operation.",
393	/* 2 */ "VMCLEAR with invalid physical address.",
394	/* 3 */ "VMCLEAR with VMXON pointer.",
395	/* 4 */ "VMLAUNCH with non-clear VMCS.",
396	/* 5 */ "VMRESUME with non-launched VMCS.",
397	/* 6 */ "VMRESUME after VMXOFF",
398	/* 7 */ "VM entry with invalid control fields.",
399	/* 8 */ "VM entry with invalid host state fields.",
400	/* 9 */ "VMPTRLD with invalid physical address.",
401	/* 10 */ "VMPTRLD with VMXON pointer.",
402	/* 11 */ "VMPTRLD with incorrect revision identifier.",
403	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
404	/* 13 */ "VMWRITE to read-only VMCS component.",
405	/* 14 */ "(Not Used)",
406	/* 15 */ "VMXON executed in VMX root operation.",
407	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
408	/* 17 */ "VM entry with non-launched executing VMCS.",
409	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
410	/* 19 */ "VMCALL with non-clear VMCS.",
411	/* 20 */ "VMCALL with invalid VM-exit control fields.",
412	/* 21 */ "(Not Used)",
413	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
414	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
415	/* 24 */ "VMCALL with invalid SMM-monitor features.",
416	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
417	/* 26 */ "VM entry with events blocked by MOV SS.",
418	/* 27 */ "(Not Used)",
419	/* 28 */ "Invalid operand to INVEPT/INVVPID."
420	};
421	#endif /* VBOX_STRICT */
422
423
424
425	/**
426	* Updates the VM's last error record. If there was a VMX instruction error,
427	* reads the error data from the VMCS and updates VCPU's last error record as
428	* well.
429	*
430	* @param pVM Pointer to the VM.
431	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
432	* VERR_VMX_UNABLE_TO_START_VM or
433	* VERR_VMX_INVALID_VMCS_FIELD).
434	* @param rc The error code.
435	*/
436	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
437	{
438	AssertPtr(pVM);
439	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
440	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
441	{
442	AssertPtrReturnVoid(pVCpu);
443	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
444	}
445	pVM->hm.s.lLastError = rc;
446	}
447
448
449	/**
450	* Reads the VM-entry interruption-information field from the VMCS into the VMX
451	* transient structure.
452	*
453	* @returns VBox status code.
454	* @param pVmxTransient Pointer to the VMX transient structure.
455	*
456	* @remarks No-long-jump zone!!!
457	*/
458	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
459	{
460	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
461	AssertRCReturn(rc, rc);
462	return VINF_SUCCESS;
463	}
464
465
466	/**
467	* Reads the VM-entry exception error code field from the VMCS into
468	* the VMX transient structure.
469	*
470	* @returns VBox status code.
471	* @param pVmxTransient Pointer to the VMX transient structure.
472	*
473	* @remarks No-long-jump zone!!!
474	*/
475	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
476	{
477	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
478	AssertRCReturn(rc, rc);
479	return VINF_SUCCESS;
480	}
481
482
483	/**
484	* Reads the VM-entry exception error code field from the VMCS into
485	* the VMX transient structure.
486	*
487	* @returns VBox status code.
488	* @param pVCpu Pointer to the VMCPU.
489	* @param pVmxTransient Pointer to the VMX transient structure.
490	*
491	* @remarks No-long-jump zone!!!
492	*/
493	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
494	{
495	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
496	AssertRCReturn(rc, rc);
497	return VINF_SUCCESS;
498	}
499
500
501	/**
502	* Reads the VM-exit interruption-information field from the VMCS into the VMX
503	* transient structure.
504	*
505	* @returns VBox status code.
506	* @param pVCpu Pointer to the VMCPU.
507	* @param pVmxTransient Pointer to the VMX transient structure.
508	*/
509	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
510	{
511	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
512	{
513	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
514	AssertRCReturn(rc, rc);
515	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
516	}
517	return VINF_SUCCESS;
518	}
519
520
521	/**
522	* Reads the VM-exit interruption error code from the VMCS into the VMX
523	* transient structure.
524	*
525	* @returns VBox status code.
526	* @param pVCpu Pointer to the VMCPU.
527	* @param pVmxTransient Pointer to the VMX transient structure.
528	*/
529	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
530	{
531	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
532	{
533	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
534	AssertRCReturn(rc, rc);
535	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
536	}
537	return VINF_SUCCESS;
538	}
539
540
541	/**
542	* Reads the VM-exit instruction length field from the VMCS into the VMX
543	* transient structure.
544	*
545	* @returns VBox status code.
546	* @param pVCpu Pointer to the VMCPU.
547	* @param pVmxTransient Pointer to the VMX transient structure.
548	*/
549	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
550	{
551	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
552	{
553	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
554	AssertRCReturn(rc, rc);
555	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
556	}
557	return VINF_SUCCESS;
558	}
559
560
561	/**
562	* Reads the exit qualification from the VMCS into the VMX transient structure.
563	*
564	* @returns VBox status code.
565	* @param pVCpu Pointer to the VMCPU.
566	* @param pVmxTransient Pointer to the VMX transient structure.
567	*/
568	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
569	{
570	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
571	{
572	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
573	AssertRCReturn(rc, rc);
574	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
575	}
576	return VINF_SUCCESS;
577	}
578
579
580	/**
581	* Reads the IDT-vectoring information field from the VMCS into the VMX
582	* transient structure.
583	*
584	* @returns VBox status code.
585	* @param pVmxTransient Pointer to the VMX transient structure.
586	*
587	* @remarks No-long-jump zone!!!
588	*/
589	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
590	{
591	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
592	{
593	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
594	AssertRCReturn(rc, rc);
595	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
596	}
597	return VINF_SUCCESS;
598	}
599
600
601	/**
602	* Reads the IDT-vectoring error code from the VMCS into the VMX
603	* transient structure.
604	*
605	* @returns VBox status code.
606	* @param pVmxTransient Pointer to the VMX transient structure.
607	*/
608	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
609	{
610	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
611	{
612	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
613	AssertRCReturn(rc, rc);
614	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
615	}
616	return VINF_SUCCESS;
617	}
618
619
620	/**
621	* Enters VMX root mode operation on the current CPU.
622	*
623	* @returns VBox status code.
624	* @param pVM Pointer to the VM (optional, can be NULL, after
625	* a resume).
626	* @param HCPhysCpuPage Physical address of the VMXON region.
627	* @param pvCpuPage Pointer to the VMXON region.
628	*/
629	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
630	{
631	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
632	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
633
634	if (pVM)
635	{
636	/* Write the VMCS revision dword to the VMXON region. */
637	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
638	}
639
640	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
641	RTCCUINTREG fFlags = ASMIntDisableFlags();
642
643	/* Enable the VMX bit in CR4 if necessary. */
644	RTCCUINTREG uCr4 = ASMGetCR4();
645	if (!(uCr4 & X86_CR4_VMXE))
646	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
647
648	/* Enter VMX root mode. */
649	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
650	if (RT_FAILURE(rc))
651	ASMSetCR4(uCr4);
652
653	/* Restore interrupts. */
654	ASMSetFlags(fFlags);
655	return rc;
656	}
657
658
659	/**
660	* Exits VMX root mode operation on the current CPU.
661	*
662	* @returns VBox status code.
663	*/
664	static int hmR0VmxLeaveRootMode(void)
665	{
666	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
667	RTCCUINTREG fFlags = ASMIntDisableFlags();
668	int rc = VINF_SUCCESS;
669
670	/* If we're for some reason not in VMX root mode, then don't leave it. */
671	if (ASMGetCR4() & X86_CR4_VMXE)
672	{
673	/* Exit VMX root mode and clear the VMX bit in CR4 */
674	VMXDisable();
675	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
676	}
677	else
678	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
679
680	/* Restore interrupts. */
681	ASMSetFlags(fFlags);
682	return rc;
683	}
684
685
686	/**
687	* Allocates and maps one physically contiguous page. The allocated page is
688	* zero'd out. (Used by various VT-x structures).
689	*
690	* @returns IPRT status code.
691	* @param pMemObj Pointer to the ring-0 memory object.
692	* @param ppVirt Where to store the virtual address of the
693	* allocation.
694	* @param pPhys Where to store the physical address of the
695	* allocation.
696	*/
697	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
698	{
699	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
700	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
701	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
702
703	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
704	if (RT_FAILURE(rc))
705	return rc;
706	ppVirt = RTR0MemObjAddress(pMemObj);
707	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
708	ASMMemZero32(*ppVirt, PAGE_SIZE);
709	return VINF_SUCCESS;
710	}
711
712
713	/**
714	* Frees and unmaps an allocated physical page.
715	*
716	* @param pMemObj Pointer to the ring-0 memory object.
717	* @param ppVirt Where to re-initialize the virtual address of
718	* allocation as 0.
719	* @param pHCPhys Where to re-initialize the physical address of the
720	* allocation as 0.
721	*/
722	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
723	{
724	AssertPtr(pMemObj);
725	AssertPtr(ppVirt);
726	AssertPtr(pHCPhys);
727	if (*pMemObj != NIL_RTR0MEMOBJ)
728	{
729	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
730	AssertRC(rc);
731	*pMemObj = NIL_RTR0MEMOBJ;
732	*ppVirt = 0;
733	*pHCPhys = 0;
734	}
735	}
736
737
738	/**
739	* Worker function to free VT-x related structures.
740	*
741	* @returns IPRT status code.
742	* @param pVM Pointer to the VM.
743	*/
744	static void hmR0VmxStructsFree(PVM pVM)
745	{
746	for (VMCPUID i = 0; i < pVM->cCpus; i++)
747	{
748	PVMCPU pVCpu = &pVM->aCpus[i];
749	AssertPtr(pVCpu);
750
751	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
752	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
753
754	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
755	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
756
757	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
758	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
759	}
760
761	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
762	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
763	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
764	#endif
765	}
766
767
768	/**
769	* Worker function to allocate VT-x related VM structures.
770	*
771	* @returns IPRT status code.
772	* @param pVM Pointer to the VM.
773	*/
774	static int hmR0VmxStructsAlloc(PVM pVM)
775	{
776	/*
777	* Initialize members up-front so we can cleanup properly on allocation failure.
778	*/
779	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
780	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
781	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
782	pVM->hm.s.vmx.HCPhys##a_Name = 0;
783
784	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
785	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
786	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
787	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
788
789	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
790	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
791	#endif
792	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
793
794	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
795	for (VMCPUID i = 0; i < pVM->cCpus; i++)
796	{
797	PVMCPU pVCpu = &pVM->aCpus[i];
798	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
799	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
800	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
801	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
802	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
803	}
804	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
805	#undef VMXLOCAL_INIT_VM_MEMOBJ
806
807	/*
808	* Allocate all the VT-x structures.
809	*/
810	int rc = VINF_SUCCESS;
811	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
812	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
813	if (RT_FAILURE(rc))
814	goto cleanup;
815	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
816	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
817	#endif
818
819	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
820	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
821	{
822	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
823	&pVM->hm.s.vmx.HCPhysApicAccess);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	}
827
828	/*
829	* Initialize per-VCPU VT-x structures.
830	*/
831	for (VMCPUID i =0; i < pVM->cCpus; i++)
832	{
833	PVMCPU pVCpu = &pVM->aCpus[i];
834	AssertPtr(pVCpu);
835
836	/* Allocate the VM control structure (VMCS). */
837	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
838	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
839	if (RT_FAILURE(rc))
840	goto cleanup;
841
842	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
843	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
844	{
845	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
846	&pVCpu->hm.s.vmx.HCPhysVirtApic);
847	if (RT_FAILURE(rc))
848	goto cleanup;
849	}
850
851	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
852	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
853	{
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
855	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
856	if (RT_FAILURE(rc))
857	goto cleanup;
858	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
859	}
860
861	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
862	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
863	if (RT_FAILURE(rc))
864	goto cleanup;
865
866	/* Allocate the VM-exit MSR-load page for the host MSRs. */
867	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
868	if (RT_FAILURE(rc))
869	goto cleanup;
870	}
871
872	return VINF_SUCCESS;
873
874	cleanup:
875	hmR0VmxStructsFree(pVM);
876	return rc;
877	}
878
879
880	/**
881	* Does global VT-x initialization (called during module initialization).
882	*
883	* @returns VBox status code.
884	*/
885	VMMR0DECL(int) VMXR0GlobalInit(void)
886	{
887	#ifdef HMVMX_USE_FUNCTION_TABLE
888	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
889	# ifdef VBOX_STRICT
890	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
891	Assert(g_apfnVMExitHandlers[i]);
892	# endif
893	#endif
894	return VINF_SUCCESS;
895	}
896
897
898	/**
899	* Does global VT-x termination (called during module termination).
900	*/
901	VMMR0DECL(void) VMXR0GlobalTerm()
902	{
903	/* Nothing to do currently. */
904	}
905
906
907	/**
908	* Sets up and activates VT-x on the current CPU.
909	*
910	* @returns VBox status code.
911	* @param pCpu Pointer to the global CPU info struct.
912	* @param pVM Pointer to the VM (can be NULL after a host resume
913	* operation).
914	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
915	* fEnabledByHost is true).
916	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
917	* @a fEnabledByHost is true).
918	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
919	* enable VT-x/AMD-V on the host.
920	*/
921	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
922	{
923	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
924	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
925
926	if (!fEnabledByHost)
927	{
928	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
929	if (RT_FAILURE(rc))
930	return rc;
931	}
932
933	/*
934	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
935	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
936	* each time while reusing a VPID after hitting the MaxASID limit once.
937	*/
938	if ( pVM
939	&& pVM->hm.s.vmx.fVpid
940	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
941	{
942	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
943	pCpu->fFlushAsidBeforeUse = false;
944	}
945	else
946	pCpu->fFlushAsidBeforeUse = true;
947
948	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
949	++pCpu->cTlbFlushes;
950
951	return VINF_SUCCESS;
952	}
953
954
955	/**
956	* Deactivates VT-x on the current CPU.
957	*
958	* @returns VBox status code.
959	* @param pCpu Pointer to the global CPU info struct.
960	* @param pvCpuPage Pointer to the VMXON region.
961	* @param HCPhysCpuPage Physical address of the VMXON region.
962	*/
963	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
964	{
965	NOREF(pCpu);
966	NOREF(pvCpuPage);
967	NOREF(HCPhysCpuPage);
968
969	hmR0VmxLeaveRootMode();
970	return VINF_SUCCESS;
971	}
972
973
974	/**
975	* Sets the permission bits for the specified MSR in the MSR bitmap.
976	*
977	* @param pVCpu Pointer to the VMCPU.
978	* @param uMSR The MSR value.
979	* @param enmRead Whether reading this MSR causes a VM-exit.
980	* @param enmWrite Whether writing this MSR causes a VM-exit.
981	*/
982	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
983	{
984	int32_t iBit;
985	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
986
987	/*
988	* Layout:
989	* 0x000 - 0x3ff - Low MSR read bits
990	* 0x400 - 0x7ff - High MSR read bits
991	* 0x800 - 0xbff - Low MSR write bits
992	* 0xc00 - 0xfff - High MSR write bits
993	*/
994	if (uMsr <= 0x00001FFF)
995	iBit = uMsr;
996	else if ( uMsr >= 0xC0000000
997	&& uMsr <= 0xC0001FFF)
998	{
999	iBit = (uMsr - 0xC0000000);
1000	pbMsrBitmap += 0x400;
1001	}
1002	else
1003	{
1004	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1005	return;
1006	}
1007
1008	Assert(iBit <= 0x1fff);
1009	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1010	ASMBitSet(pbMsrBitmap, iBit);
1011	else
1012	ASMBitClear(pbMsrBitmap, iBit);
1013
1014	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1015	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1016	else
1017	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1018	}
1019
1020
1021	/**
1022	* Flushes the TLB using EPT.
1023	*
1024	* @returns VBox status code.
1025	* @param pVM Pointer to the VM.
1026	* @param pVCpu Pointer to the VMCPU.
1027	* @param enmFlush Type of flush.
1028	*/
1029	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1030	{
1031	AssertPtr(pVM);
1032	Assert(pVM->hm.s.fNestedPaging);
1033
1034	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1035
1036	uint64_t descriptor[2];
1037	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1038	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1039
1040	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1041	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1042	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1043	}
1044
1045
1046	/**
1047	* Flushes the TLB using VPID.
1048	*
1049	* @returns VBox status code.
1050	* @param pVM Pointer to the VM.
1051	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1052	* enmFlush).
1053	* @param enmFlush Type of flush.
1054	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1055	* on @a enmFlush).
1056	*/
1057	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1058	{
1059	AssertPtr(pVM);
1060	Assert(pVM->hm.s.vmx.fVpid);
1061
1062	uint64_t descriptor[2];
1063	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1064	{
1065	descriptor[0] = 0;
1066	descriptor[1] = 0;
1067	}
1068	else
1069	{
1070	AssertPtr(pVCpu);
1071	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1072	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1073	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1074	descriptor[1] = GCPtr;
1075	}
1076
1077	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1078	AssertMsg(rc == VINF_SUCCESS,
1079	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1080	if ( RT_SUCCESS(rc)
1081	&& pVCpu)
1082	{
1083	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1084	}
1085	}
1086
1087
1088	/**
1089	* Invalidates a guest page by guest virtual address. Only relevant for
1090	* EPT/VPID, otherwise there is nothing really to invalidate.
1091	*
1092	* @returns VBox status code.
1093	* @param pVM Pointer to the VM.
1094	* @param pVCpu Pointer to the VMCPU.
1095	* @param GCVirt Guest virtual address of the page to invalidate.
1096	*/
1097	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1098	{
1099	AssertPtr(pVM);
1100	AssertPtr(pVCpu);
1101	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1102
1103	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1104	if (!fFlushPending)
1105	{
1106	/*
1107	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1108	* See @bugref{6043} and @bugref{6177}.
1109	*
1110	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1111	* function maybe called in a loop with individual addresses.
1112	*/
1113	if (pVM->hm.s.vmx.fVpid)
1114	{
1115	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1116	{
1117	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1118	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1119	}
1120	else
1121	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1122	}
1123	else if (pVM->hm.s.fNestedPaging)
1124	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1125	}
1126
1127	return VINF_SUCCESS;
1128	}
1129
1130
1131	/**
1132	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1133	* otherwise there is nothing really to invalidate.
1134	*
1135	* @returns VBox status code.
1136	* @param pVM Pointer to the VM.
1137	* @param pVCpu Pointer to the VMCPU.
1138	* @param GCPhys Guest physical address of the page to invalidate.
1139	*/
1140	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1141	{
1142	LogFlowFunc(("%RGp\n", GCPhys));
1143
1144	/*
1145	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1146	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1147	* This function might be called in a loop.
1148	*/
1149	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1150	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1151	return VINF_SUCCESS;
1152	}
1153
1154
1155	/**
1156	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1157	* case where neither EPT nor VPID is supported by the CPU.
1158	*
1159	* @param pVM Pointer to the VM.
1160	* @param pVCpu Pointer to the VMCPU.
1161	*
1162	* @remarks Called with interrupts disabled.
1163	*/
1164	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1165	{
1166	NOREF(pVM);
1167	AssertPtr(pVCpu);
1168	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1169	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1170
1171	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1172	AssertPtr(pCpu);
1173
1174	pVCpu->hm.s.TlbShootdown.cPages = 0;
1175	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1176	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1177	pVCpu->hm.s.fForceTLBFlush = false;
1178	return;
1179	}
1180
1181
1182	/**
1183	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1184	*
1185	* @param pVM Pointer to the VM.
1186	* @param pVCpu Pointer to the VMCPU.
1187	* @remarks All references to "ASID" in this function pertains to "VPID" in
1188	* Intel's nomenclature. The reason is, to avoid confusion in compare
1189	* statements since the host-CPU copies are named "ASID".
1190	*
1191	* @remarks Called with interrupts disabled.
1192	*/
1193	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1194	{
1195	AssertPtr(pVM);
1196	AssertPtr(pVCpu);
1197	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1198	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1199	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1200
1201	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1202	AssertPtr(pCpu);
1203
1204	/*
1205	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1206	* This can happen both for start & resume due to long jumps back to ring-3.
1207	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1208	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1209	*/
1210	bool fNewASID = false;
1211	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1212	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1213	{
1214	pVCpu->hm.s.fForceTLBFlush = true;
1215	fNewASID = true;
1216	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1217	}
1218
1219	/*
1220	* Check for explicit TLB shootdowns.
1221	*/
1222	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1223	{
1224	pVCpu->hm.s.fForceTLBFlush = true;
1225	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1226	}
1227
1228	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1229	if (pVCpu->hm.s.fForceTLBFlush)
1230	{
1231	if (fNewASID)
1232	{
1233	++pCpu->uCurrentAsid;
1234	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1235	{
1236	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1237	pCpu->cTlbFlushes++;
1238	pCpu->fFlushAsidBeforeUse = true;
1239	}
1240
1241	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1242	if (pCpu->fFlushAsidBeforeUse)
1243	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1244	}
1245	else
1246	{
1247	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1248	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1249	else
1250	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1251	}
1252
1253	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1254	pVCpu->hm.s.fForceTLBFlush = false;
1255	}
1256	else
1257	{
1258	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1259	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1260	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1261	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1262
1263	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1264	* not be executed. See hmQueueInvlPage() where it is commented
1265	* out. Support individual entry flushing someday. */
1266	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1267	{
1268	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1269
1270	/*
1271	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1272	* as supported by the CPU.
1273	*/
1274	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1275	{
1276	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1277	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1278	}
1279	else
1280	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1281	}
1282	else
1283	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1284	}
1285	pVCpu->hm.s.TlbShootdown.cPages = 0;
1286	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1287
1288	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1289	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1290	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1291	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1292	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1293	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1294
1295	/* Update VMCS with the VPID. */
1296	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1297	AssertRC(rc);
1298	}
1299
1300
1301	/**
1302	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1303	*
1304	* @returns VBox status code.
1305	* @param pVM Pointer to the VM.
1306	* @param pVCpu Pointer to the VMCPU.
1307	*
1308	* @remarks Called with interrupts disabled.
1309	*/
1310	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1311	{
1312	AssertPtr(pVM);
1313	AssertPtr(pVCpu);
1314	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1315	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1316
1317	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1318	AssertPtr(pCpu);
1319
1320	/*
1321	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1322	* This can happen both for start & resume due to long jumps back to ring-3.
1323	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1324	*/
1325	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1326	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1327	{
1328	pVCpu->hm.s.fForceTLBFlush = true;
1329	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1330	}
1331
1332	/* Check for explicit TLB shootdown flushes. */
1333	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1334	{
1335	pVCpu->hm.s.fForceTLBFlush = true;
1336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1337	}
1338
1339	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1340	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1341
1342	if (pVCpu->hm.s.fForceTLBFlush)
1343	{
1344	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1345	pVCpu->hm.s.fForceTLBFlush = false;
1346	}
1347	else
1348	{
1349	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1350	* not be executed. See hmQueueInvlPage() where it is commented
1351	* out. Support individual entry flushing someday. */
1352	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1353	{
1354	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1355	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1356	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1357	}
1358	else
1359	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1360	}
1361
1362	pVCpu->hm.s.TlbShootdown.cPages = 0;
1363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1364	}
1365
1366
1367	/**
1368	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1369	*
1370	* @returns VBox status code.
1371	* @param pVM Pointer to the VM.
1372	* @param pVCpu Pointer to the VMCPU.
1373	*
1374	* @remarks Called with interrupts disabled.
1375	*/
1376	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1377	{
1378	AssertPtr(pVM);
1379	AssertPtr(pVCpu);
1380	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1381	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1382
1383	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1384
1385	/*
1386	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1387	* This can happen both for start & resume due to long jumps back to ring-3.
1388	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1389	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1390	*/
1391	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1392	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1393	{
1394	pVCpu->hm.s.fForceTLBFlush = true;
1395	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1396	}
1397
1398	/* Check for explicit TLB shootdown flushes. */
1399	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1400	{
1401	/*
1402	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1403	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1404	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1405	*/
1406	pVCpu->hm.s.fForceTLBFlush = true;
1407	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1408	}
1409
1410	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1411	if (pVCpu->hm.s.fForceTLBFlush)
1412	{
1413	++pCpu->uCurrentAsid;
1414	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1415	{
1416	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1417	pCpu->fFlushAsidBeforeUse = true;
1418	pCpu->cTlbFlushes++;
1419	}
1420
1421	pVCpu->hm.s.fForceTLBFlush = false;
1422	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1423	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1424	if (pCpu->fFlushAsidBeforeUse)
1425	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1426	}
1427	else
1428	{
1429	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1430	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1431	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1432	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1433
1434	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1435	* not be executed. See hmQueueInvlPage() where it is commented
1436	* out. Support individual entry flushing someday. */
1437	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1438	{
1439	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1440	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1441	{
1442	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1443	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1444	}
1445	else
1446	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1447	}
1448	else
1449	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1450	}
1451
1452	pVCpu->hm.s.TlbShootdown.cPages = 0;
1453	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1454
1455	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1456	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1457	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1458	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1459	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1460	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1461
1462	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1463	AssertRC(rc);
1464	}
1465
1466
1467	/**
1468	* Flushes the guest TLB entry based on CPU capabilities.
1469	*
1470	* @param pVCpu Pointer to the VMCPU.
1471	*/
1472	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1473	{
1474	PVM pVM = pVCpu->CTX_SUFF(pVM);
1475	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1476	{
1477	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1478	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1479	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1480	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1481	default:
1482	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1483	break;
1484	}
1485	}
1486
1487
1488	/**
1489	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1490	* TLB entries from the host TLB before VM-entry.
1491	*
1492	* @returns VBox status code.
1493	* @param pVM Pointer to the VM.
1494	*/
1495	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1496	{
1497	/*
1498	* Determine optimal flush type for nested paging.
1499	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1500	* guest execution (see hmR3InitFinalizeR0()).
1501	*/
1502	if (pVM->hm.s.fNestedPaging)
1503	{
1504	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1505	{
1506	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1507	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1508	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1509	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1510	else
1511	{
1512	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1513	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1514	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1515	}
1516
1517	/* Make sure the write-back cacheable memory type for EPT is supported. */
1518	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1519	{
1520	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1521	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1522	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1523	}
1524	}
1525	else
1526	{
1527	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1528	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1529	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1530	}
1531	}
1532
1533	/*
1534	* Determine optimal flush type for VPID.
1535	*/
1536	if (pVM->hm.s.vmx.fVpid)
1537	{
1538	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1539	{
1540	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1541	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1542	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1543	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1544	else
1545	{
1546	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1547	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1548	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1549	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1550	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1552	pVM->hm.s.vmx.fVpid = false;
1553	}
1554	}
1555	else
1556	{
1557	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1558	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1559	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1560	pVM->hm.s.vmx.fVpid = false;
1561	}
1562	}
1563
1564	/*
1565	* Setup the handler for flushing tagged-TLBs.
1566	*/
1567	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1568	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1569	else if (pVM->hm.s.fNestedPaging)
1570	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1571	else if (pVM->hm.s.vmx.fVpid)
1572	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1573	else
1574	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1575	return VINF_SUCCESS;
1576	}
1577
1578
1579	/**
1580	* Sets up pin-based VM-execution controls in the VMCS.
1581	*
1582	* @returns VBox status code.
1583	* @param pVM Pointer to the VM.
1584	* @param pVCpu Pointer to the VMCPU.
1585	*/
1586	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1587	{
1588	AssertPtr(pVM);
1589	AssertPtr(pVCpu);
1590
1591	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1592	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1593
1594	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1595	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1596	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1597
1598	/* Enable the VMX preemption timer. */
1599	if (pVM->hm.s.vmx.fUsePreemptTimer)
1600	{
1601	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1602	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1603	}
1604
1605	if ((val & zap) != val)
1606	{
1607	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1608	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1609	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1610	}
1611
1612	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1613	AssertRCReturn(rc, rc);
1614
1615	/* Update VCPU with the currently set pin-based VM-execution controls. */
1616	pVCpu->hm.s.vmx.u32PinCtls = val;
1617	return rc;
1618	}
1619
1620
1621	/**
1622	* Sets up processor-based VM-execution controls in the VMCS.
1623	*
1624	* @returns VBox status code.
1625	* @param pVM Pointer to the VM.
1626	* @param pVMCPU Pointer to the VMCPU.
1627	*/
1628	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1629	{
1630	AssertPtr(pVM);
1631	AssertPtr(pVCpu);
1632
1633	int rc = VERR_INTERNAL_ERROR_5;
1634	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1635	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1636
1637	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1638	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1639	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1640	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1641	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1642	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1643	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1644
1645	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1646	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1647	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1648	{
1649	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1654	if (!pVM->hm.s.fNestedPaging)
1655	{
1656	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1657	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1658	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1659	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1660	}
1661
1662	/* Use TPR shadowing if supported by the CPU. */
1663	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1664	{
1665	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1666	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1667	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1668	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1669	AssertRCReturn(rc, rc);
1670
1671	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1672	/* CR8 writes causes a VM-exit based on TPR threshold. */
1673	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1674	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1675	}
1676	else
1677	{
1678	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1679	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1680	}
1681
1682	/* Use MSR-bitmaps if supported by the CPU. */
1683	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1684	{
1685	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1686
1687	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1688	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1689	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1690	AssertRCReturn(rc, rc);
1691
1692	/*
1693	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1694	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1695	*/
1696	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1697	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1698	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1699	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1700	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1701	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1702	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1703	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1704	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1705	}
1706
1707	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1708	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1709	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1710
1711	if ((val & zap) != val)
1712	{
1713	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1714	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1715	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1716	}
1717
1718	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1719	AssertRCReturn(rc, rc);
1720
1721	/* Update VCPU with the currently set processor-based VM-execution controls. */
1722	pVCpu->hm.s.vmx.u32ProcCtls = val;
1723
1724	/*
1725	* Secondary processor-based VM-execution controls.
1726	*/
1727	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1728	{
1729	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1730	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1731
1732	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1733	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1734
1735	if (pVM->hm.s.fNestedPaging)
1736	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1737	else
1738	{
1739	/*
1740	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1741	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1742	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1743	*/
1744	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1745	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1746	}
1747
1748	if (pVM->hm.s.vmx.fVpid)
1749	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1750
1751	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1752	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1753
1754	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1755	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1756	* done dynamically. */
1757	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1758	{
1759	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1760	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1761	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1762	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1763	AssertRCReturn(rc, rc);
1764	}
1765
1766	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1767	{
1768	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1769	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1770	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1771	}
1772
1773	if ((val & zap) != val)
1774	{
1775	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1776	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1777	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1778	}
1779
1780	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1781	AssertRCReturn(rc, rc);
1782
1783	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1784	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1785	}
1786
1787	return VINF_SUCCESS;
1788	}
1789
1790
1791	/**
1792	* Sets up miscellaneous (everything other than Pin & Processor-based
1793	* VM-execution) control fields in the VMCS.
1794	*
1795	* @returns VBox status code.
1796	* @param pVM Pointer to the VM.
1797	* @param pVCpu Pointer to the VMCPU.
1798	*/
1799	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1800	{
1801	AssertPtr(pVM);
1802	AssertPtr(pVCpu);
1803
1804	int rc = VERR_GENERAL_FAILURE;
1805
1806	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1807	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1808
1809	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1810
1811	/*
1812	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1813	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1814	* We thus use the exception bitmap to control it rather than use both.
1815	*/
1816	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1817	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1818
1819	/** @todo Explore possibility of using IO-bitmaps. */
1820	/* All IO & IOIO instructions cause VM-exits. */
1821	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1823
1824	/* Setup MSR autoloading/autostoring. */
1825	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1826	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1827	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1829	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1830	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1831
1832	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1833	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1834	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1835	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1836
1837	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1838	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1839
1840	/* Setup debug controls */
1841	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1842	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1843	AssertRCReturn(rc, rc);
1844	return rc;
1845	}
1846
1847
1848	/**
1849	* Sets up the initial exception bitmap in the VMCS based on static conditions
1850	* (i.e. conditions that cannot ever change at runtime).
1851	*
1852	* @returns VBox status code.
1853	* @param pVM Pointer to the VM.
1854	* @param pVCpu Pointer to the VMCPU.
1855	*/
1856	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1857	{
1858	AssertPtr(pVM);
1859	AssertPtr(pVCpu);
1860
1861	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1862
1863	uint32_t u32XcptBitmap = 0;
1864
1865	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1866	if (!pVM->hm.s.fNestedPaging)
1867	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1868
1869	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1870	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1871	AssertRCReturn(rc, rc);
1872	return rc;
1873	}
1874
1875
1876	/**
1877	* Sets up the initial guest-state mask. The guest-state mask is consulted
1878	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1879	* for the nested virtualization case (as it would cause a VM-exit).
1880	*
1881	* @param pVCpu Pointer to the VMCPU.
1882	*/
1883	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1884	{
1885	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1886	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Does per-VM VT-x initialization.
1893	*
1894	* @returns VBox status code.
1895	* @param pVM Pointer to the VM.
1896	*/
1897	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1898	{
1899	LogFlowFunc(("pVM=%p\n", pVM));
1900
1901	int rc = hmR0VmxStructsAlloc(pVM);
1902	if (RT_FAILURE(rc))
1903	{
1904	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1905	return rc;
1906	}
1907
1908	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1909	{
1910	PVMCPU pVCpu = &pVM->aCpus[i];
1911
1912	/* Current guest paging mode. */
1913	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1914	}
1915
1916	return VINF_SUCCESS;
1917	}
1918
1919
1920	/**
1921	* Does per-VM VT-x termination.
1922	*
1923	* @returns VBox status code.
1924	* @param pVM Pointer to the VM.
1925	*/
1926	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1927	{
1928	LogFlowFunc(("pVM=%p\n", pVM));
1929
1930	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1931	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1932	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1933	#endif
1934	hmR0VmxStructsFree(pVM);
1935	return VINF_SUCCESS;
1936	}
1937
1938
1939	/**
1940	* Sets up the VM for execution under VT-x.
1941	* This function is only called once per-VM during initalization.
1942	*
1943	* @returns VBox status code.
1944	* @param pVM Pointer to the VM.
1945	*/
1946	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1947	{
1948	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1949	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1950
1951	LogFlowFunc(("pVM=%p\n", pVM));
1952
1953	/*
1954	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1955	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1956	*/
1957	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1958	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1959	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1960	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1961	{
1962	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1963	return VERR_INTERNAL_ERROR;
1964	}
1965
1966	/* Initialize these always, see hmR3InitFinalizeR0().*/
1967	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1968	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1969
1970	/* Setup the tagged-TLB flush handlers. */
1971	int rc = hmR0VmxSetupTaggedTlb(pVM);
1972	if (RT_FAILURE(rc))
1973	{
1974	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1975	return rc;
1976	}
1977
1978	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1979	{
1980	PVMCPU pVCpu = &pVM->aCpus[i];
1981	AssertPtr(pVCpu);
1982	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1983
1984	/* Set revision dword at the beginning of the VMCS structure. */
1985	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1986
1987	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1988	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1989	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1990	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1991
1992	/* Load this VMCS as the current VMCS. */
1993	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1994	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1995	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1996
1997	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1998	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1999	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2000
2001	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2002	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2003	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2004
2005	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2006	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2007	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2008
2009	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2010	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2011	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2012
2013	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2014	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2015	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2016
2017	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2018	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2019	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2020	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2021	#endif
2022
2023	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2024	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2029	}
2030
2031	return VINF_SUCCESS;
2032	}
2033
2034
2035	/**
2036	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2037	* the VMCS.
2038	*
2039	* @returns VBox status code.
2040	* @param pVM Pointer to the VM.
2041	* @param pVCpu Pointer to the VMCPU.
2042	*/
2043	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2044	{
2045	RTCCUINTREG uReg = ASMGetCR0();
2046	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2047
2048	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2049	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2050	if (HMVMX_IS_64BIT_HOST_MODE())
2051	{
2052	uint64_t uReg = hmR0Get64bitCR3();
2053	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2054	}
2055	else
2056	#endif
2057	{
2058	uReg = ASMGetCR3();
2059	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2060	}
2061
2062	uReg = ASMGetCR4();
2063	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2064	AssertRCReturn(rc, rc);
2065	return rc;
2066	}
2067
2068
2069	/**
2070	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2071	* the host-state area in the VMCS.
2072	*
2073	* @returns VBox status code.
2074	* @param pVM Pointer to the VM.
2075	* @param pVCpu Pointer to the VMCPU.
2076	*/
2077	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2078	{
2079	int rc = VERR_INTERNAL_ERROR_5;
2080	RTSEL uSelCS = 0;
2081	RTSEL uSelSS = 0;
2082	RTSEL uSelDS = 0;
2083	RTSEL uSelES = 0;
2084	RTSEL uSelFS = 0;
2085	RTSEL uSelGS = 0;
2086	RTSEL uSelTR = 0;
2087
2088	/*
2089	* Host Selector registers.
2090	*/
2091	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2092	if (HMVMX_IS_64BIT_HOST_MODE())
2093	{
2094	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2095	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2096	}
2097	else
2098	{
2099	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2100	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2101	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2102	}
2103	#else
2104	uSelCS = ASMGetCS();
2105	uSelSS = ASMGetSS();
2106	#endif
2107
2108	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2109	/** @todo Verify if we have any platform that actually run with DS or ES with
2110	* RPL != 0 in kernel space. */
2111	uSelDS = 0;
2112	uSelES = 0;
2113	uSelFS = 0;
2114	uSelGS = 0;
2115	uSelTR = ASMGetTR();
2116
2117	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2118	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2119	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2120	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2121	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2122	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2123	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2124	Assert(uSelCS != 0);
2125	Assert(uSelTR != 0);
2126
2127	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2128	#if 0
2129	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2130	Assert(uSelSS != 0);
2131	#endif
2132
2133	/* Write these host selector fields into the host-state area in the VMCS. */
2134	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2135	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2136	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2137	#if 0
2138	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2139	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2140	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2141	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2142	#endif
2143	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2144	AssertRCReturn(rc, rc);
2145
2146	/*
2147	* Host GDTR and IDTR.
2148	*/
2149	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2150	* be safe to -not- save and restore GDTR and IDTR in the assembly
2151	* code and just do it here and don't care if the limits are zapped on
2152	* VM-exit. */
2153	RTGDTR Gdtr;
2154	RT_ZERO(Gdtr);
2155	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2156	if (HMVMX_IS_64BIT_HOST_MODE())
2157	{
2158	X86XDTR64 Gtr64;
2159	X86XDTR64 Idtr64;
2160	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2161	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2162	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2163	Gdtr.cbGdt = Gdtr64.cb;
2164	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2165	}
2166	else
2167	#endif
2168	{
2169	RTIDTR Idtr;
2170	ASMGetGDTR(&Gdtr);
2171	ASMGetIDTR(&Idtr);
2172	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2173	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2174	}
2175	AssertRCReturn(rc, rc);
2176
2177	/*
2178	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2179	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2180	*/
2181	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2182	{
2183	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2184	return VERR_VMX_INVALID_HOST_STATE;
2185	}
2186
2187	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2188	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2189	if (HMVMX_IS_64BIT_HOST_MODE())
2190	{
2191	/* We need the 64-bit TR base for hybrid darwin. */
2192	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2193	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2194	}
2195	else
2196	#endif
2197	{
2198	uintptr_t uTRBase;
2199	#if HC_ARCH_BITS == 64
2200	uTRBase = X86DESC64_BASE(pDesc);
2201	#else
2202	uTRBase = X86DESC_BASE(pDesc);
2203	#endif
2204	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2205	}
2206	AssertRCReturn(rc, rc);
2207
2208	/*
2209	* Host FS base and GS base.
2210	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2211	* would take care of the bases. In 64-bit, the MSRs come into play.
2212	*/
2213	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2214	if (HMVMX_IS_64BIT_HOST_MODE())
2215	{
2216	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2217	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2218	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2219	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2220	AssertRCReturn(rc, rc);
2221	}
2222	#endif
2223	return rc;
2224	}
2225
2226
2227	/**
2228	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2229	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2230	* the host after every successful VM exit.
2231	*
2232	* @returns VBox status code.
2233	* @param pVM Pointer to the VM.
2234	* @param pVCpu Pointer to the VMCPU.
2235	*/
2236	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2237	{
2238	AssertPtr(pVCpu);
2239	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2240
2241	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2242	uint32_t cHostMsrs = 0;
2243	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2244
2245	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2246	{
2247	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2248	pHostMsr->u32Reserved = 0;
2249	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2250	if (CPUMIsGuestInLongMode(pVCpu))
2251	{
2252	/* Must match the EFER value in our 64 bits switcher. */
2253	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2254	}
2255	else
2256	#endif
2257	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2258	pHostMsr++; cHostMsrs++;
2259	}
2260
2261	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2262	if (HMVMX_IS_64BIT_HOST_MODE())
2263	{
2264	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2265	pHostMsr->u32Reserved = 0;
2266	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2267	pHostMsr++; cHostMsrs++;
2268	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2269	pHostMsr->u32Reserved = 0;
2270	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2271	pHostMsr++; cHostMsrs++;
2272	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2273	pHostMsr->u32Reserved = 0;
2274	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2275	pHostMsr++; cHostMsrs++;
2276	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2277	#if 0
2278	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2279	pMsr->u32Reserved = 0;
2280	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2281	pHostMsr++; cHostMsrs++;
2282	#endif
2283	}
2284	#endif
2285
2286	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2287	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2288	{
2289	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2290	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2291	}
2292
2293	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2294
2295	/*
2296	* Host Sysenter MSRs.
2297	*/
2298	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2299	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2300	if (HMVMX_IS_64BIT_HOST_MODE())
2301	{
2302	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2303	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2304	}
2305	else
2306	{
2307	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2308	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2309	}
2310	#elif HC_ARCH_BITS == 32
2311	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2312	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2313	#else
2314	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2315	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2316	#endif
2317	AssertRCReturn(rc, rc);
2318
2319	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2320	* hmR0VmxSetupExitCtls() !! */
2321	return rc;
2322	}
2323
2324
2325	/**
2326	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2327	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2328	* controls".
2329	*
2330	* @returns VBox status code.
2331	* @param pVCpu Pointer to the VMCPU.
2332	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2333	* out-of-sync. Make sure to update the required fields
2334	* before using them.
2335	*
2336	* @remarks No-long-jump zone!!!
2337	*/
2338	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2339	{
2340	int rc = VINF_SUCCESS;
2341	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2342	{
2343	PVM pVM = pVCpu->CTX_SUFF(pVM);
2344	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2345	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2346
2347	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2348	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2349
2350	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2351	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2352	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2353	else
2354	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2355
2356	/*
2357	* The following should not be set (since we're not in SMM mode):
2358	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2359	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2360	*/
2361
2362	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2363	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2364	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2365
2366	if ((val & zap) != val)
2367	{
2368	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2369	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2370	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2371	}
2372
2373	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2374	AssertRCReturn(rc, rc);
2375
2376	/* Update VCPU with the currently set VM-exit controls. */
2377	pVCpu->hm.s.vmx.u32EntryCtls = val;
2378	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2379	}
2380	return rc;
2381	}
2382
2383
2384	/**
2385	* Sets up the VM-exit controls in the VMCS.
2386	*
2387	* @returns VBox status code.
2388	* @param pVM Pointer to the VM.
2389	* @param pVCpu Pointer to the VMCPU.
2390	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2391	* out-of-sync. Make sure to update the required fields
2392	* before using them.
2393	*
2394	* @remarks requires EFER.
2395	*/
2396	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2397	{
2398	int rc = VINF_SUCCESS;
2399	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2400	{
2401	PVM pVM = pVCpu->CTX_SUFF(pVM);
2402	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2403	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2404
2405	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2406	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2407
2408	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2409	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2410	if (HMVMX_IS_64BIT_HOST_MODE())
2411	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2412	else
2413	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2414	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2415	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2416	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2417	else
2418	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2419	#endif
2420
2421	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2422	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2423
2424	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2425	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2426	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2427	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2428	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2429
2430	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2431	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2432
2433	if ((val & zap) != val)
2434	{
2435	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2436	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2437	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2438	}
2439
2440	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2441	AssertRCReturn(rc, rc);
2442
2443	/* Update VCPU with the currently set VM-exit controls. */
2444	pVCpu->hm.s.vmx.u32ExitCtls = val;
2445	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2446	}
2447	return rc;
2448	}
2449
2450
2451	/**
2452	* Loads the guest APIC and related state.
2453	*
2454	* @returns VBox status code.
2455	* @param pVM Pointer to the VM.
2456	* @param pVCpu Pointer to the VMCPU.
2457	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2458	* out-of-sync. Make sure to update the required fields
2459	* before using them.
2460	*/
2461	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2462	{
2463	int rc = VINF_SUCCESS;
2464	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2465	{
2466	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2467	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2468	{
2469	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2470
2471	bool fPendingIntr = false;
2472	uint8_t u8GuestTpr = 0;
2473	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2474	AssertRCReturn(rc, rc);
2475
2476	/*
2477	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2478	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2479	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2480	* the interrupt when we VM-exit for other reasons.
2481	*/
2482	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2483	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2484	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2485	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2486
2487	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2488	AssertRCReturn(rc, rc);
2489
2490	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2491	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2492	{
2493	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2494	pMixedCtx->msrLSTAR = u8GuestTpr;
2495	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2496	{
2497	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2498	if (fPendingIntr)
2499	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2500	else
2501	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2502	}
2503	}
2504	}
2505
2506	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2507	}
2508	return rc;
2509	}
2510
2511
2512	/**
2513	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2514	*
2515	* @returns
2516	* @param pVCpu Pointer to the VMCPU.
2517	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2518	* out-of-sync. Make sure to update the required fields
2519	* before using them.
2520	*
2521	* @remarks No-long-jump zone!!!
2522	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2523	*/
2524	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2525	{
2526	/*
2527	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2528	* inhibit interrupts or clear any existing interrupt-inhibition.
2529	*/
2530	uint32_t uIntrState = 0;
2531	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2532	{
2533	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2534	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2535	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2536	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2537	{
2538	/*
2539	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2540	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2541	*/
2542	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2543	}
2544	else if (pMixedCtx->eflags.Bits.u1IF)
2545	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2546	else
2547	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2548	}
2549	return uIntrState;
2550	}
2551
2552
2553	/**
2554	* Loads the guest's interruptibility-state into the guest-state area in the
2555	* VMCS.
2556	*
2557	* @returns VBox status code.
2558	* @param pVCpu Pointer to the VMCPU.
2559	* @param uIntrState The interruptibility-state to set.
2560	*/
2561	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2562	{
2563	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2564	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2565	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2566	AssertRCReturn(rc, rc);
2567	return rc;
2568	}
2569
2570
2571	/**
2572	* Loads the guest's RIP into the guest-state area in the VMCS.
2573	*
2574	* @returns VBox status code.
2575	* @param pVCpu Pointer to the VMCPU.
2576	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2577	* out-of-sync. Make sure to update the required fields
2578	* before using them.
2579	*
2580	* @remarks No-long-jump zone!!!
2581	*/
2582	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2583	{
2584	int rc = VINF_SUCCESS;
2585	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2586	{
2587	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2588	AssertRCReturn(rc, rc);
2589	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2590	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2591	}
2592	return rc;
2593	}
2594
2595
2596	/**
2597	* Loads the guest's RSP into the guest-state area in the VMCS.
2598	*
2599	* @returns VBox status code.
2600	* @param pVCpu Pointer to the VMCPU.
2601	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2602	* out-of-sync. Make sure to update the required fields
2603	* before using them.
2604	*
2605	* @remarks No-long-jump zone!!!
2606	*/
2607	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2608	{
2609	int rc = VINF_SUCCESS;
2610	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2611	{
2612	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2613	AssertRCReturn(rc, rc);
2614	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2615	}
2616	return rc;
2617	}
2618
2619
2620	/**
2621	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2622	*
2623	* @returns VBox status code.
2624	* @param pVCpu Pointer to the VMCPU.
2625	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2626	* out-of-sync. Make sure to update the required fields
2627	* before using them.
2628	*
2629	* @remarks No-long-jump zone!!!
2630	*/
2631	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2632	{
2633	int rc = VINF_SUCCESS;
2634	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2635	{
2636	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2637	Let us assert it as such and use 32-bit VMWRITE. */
2638	Assert(!(pMixedCtx->rflags.u64 >> 32));
2639	X86EFLAGS uEFlags = pMixedCtx->eflags;
2640	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2641	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2642
2643	/*
2644	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2645	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2646	*/
2647	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2648	{
2649	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2650	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2651	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2652	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2653	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2654	}
2655
2656	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2657	AssertRCReturn(rc, rc);
2658
2659	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2661	}
2662	return rc;
2663	}
2664
2665
2666	/**
2667	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2668	*
2669	* @returns VBox status code.
2670	* @param pVCpu Pointer to the VMCPU.
2671	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2672	* out-of-sync. Make sure to update the required fields
2673	* before using them.
2674	*
2675	* @remarks No-long-jump zone!!!
2676	*/
2677	static int hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2678	{
2679	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2680	rc \|= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2681	rc \|= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2682	return rc;
2683	}
2684
2685
2686	/**
2687	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2688	* in the VMCS.
2689	*
2690	* @returns VBox status code.
2691	* @param pVM Pointer to the VM.
2692	* @param pVCpu Pointer to the VMCPU.
2693	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2694	* out-of-sync. Make sure to update the required fields
2695	* before using them.
2696	*
2697	* @remarks No-long-jump zone!!!
2698	*/
2699	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2700	{
2701	int rc = VINF_SUCCESS;
2702	PVM pVM = pVCpu->CTX_SUFF(pVM);
2703
2704	/*
2705	* Guest CR0.
2706	* Guest FPU.
2707	*/
2708	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2709	{
2710	Assert(!(pCtx->cr0 >> 32));
2711	uint32_t u32GuestCR0 = pCtx->cr0;
2712
2713	/* The guest's view (read access) of its CR0 is unblemished. */
2714	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2715	AssertRCReturn(rc, rc);
2716	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2717
2718	/* Setup VT-x's view of the guest CR0. */
2719	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2720	if (pVM->hm.s.fNestedPaging)
2721	{
2722	if (CPUMIsGuestPagingEnabledEx(pCtx))
2723	{
2724	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2725	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2726	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2727	}
2728	else
2729	{
2730	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2731	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2732	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2733	}
2734
2735	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2736	AssertRCReturn(rc, rc);
2737	}
2738	else
2739	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2740
2741	/*
2742	* Guest FPU bits.
2743	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2744	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2745	*/
2746	u32GuestCR0 \|= X86_CR0_NE;
2747	bool fInterceptNM = false;
2748	if (CPUMIsGuestFPUStateActive(pVCpu))
2749	{
2750	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2751	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2752	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2753	}
2754	else
2755	{
2756	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2757	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2758	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2759	}
2760
2761	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2762	bool fInterceptMF = false;
2763	if (!(pCtx->cr0 & X86_CR0_NE))
2764	fInterceptMF = true;
2765
2766	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2767	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2768	{
2769	Assert(PDMVmmDevHeapIsEnabled(pVM));
2770	Assert(pVM->hm.s.vmx.pRealModeTSS);
2771	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2772	fInterceptNM = true;
2773	fInterceptMF = true;
2774	}
2775	else
2776	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2777
2778	if (fInterceptNM)
2779	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2780	else
2781	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2782
2783	if (fInterceptMF)
2784	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2785	else
2786	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2787
2788	/* Additional intercepts for debugging, define these yourself explicitly. */
2789	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2790	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2791	\| RT_BIT(X86_XCPT_DB)
2792	\| RT_BIT(X86_XCPT_DE)
2793	\| RT_BIT(X86_XCPT_NM)
2794	\| RT_BIT(X86_XCPT_UD)
2795	\| RT_BIT(X86_XCPT_NP)
2796	\| RT_BIT(X86_XCPT_SS)
2797	\| RT_BIT(X86_XCPT_GP)
2798	\| RT_BIT(X86_XCPT_PF)
2799	\| RT_BIT(X86_XCPT_MF);
2800	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2801	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2802	#endif
2803
2804	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2805
2806	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2807	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2808	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2809	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2810	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2811	else
2812	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2813
2814	u32GuestCR0 \|= uSetCR0;
2815	u32GuestCR0 &= uZapCR0;
2816	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2817
2818	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2819	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2820	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2821	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2822
2823	/*
2824	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2825	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2826	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2827	*/
2828	uint64_t u64CR0Mask = 0;
2829	u64CR0Mask = X86_CR0_PE
2830	\| X86_CR0_NE
2831	\| X86_CR0_WP
2832	\| X86_CR0_PG
2833	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2834	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2835	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2836	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2837	u64CR0Mask &= ~X86_CR0_PE;
2838	if (pVM->hm.s.fNestedPaging)
2839	u64CR0Mask &= ~X86_CR0_WP;
2840
2841	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2842	if (fInterceptNM)
2843	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2844	else
2845	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2846
2847	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2848	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2849	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2850	AssertRCReturn(rc, rc);
2851
2852	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2853	}
2854
2855	/*
2856	* Guest CR2.
2857	* It's always loaded in the assembler code. Nothing to do here.
2858	*/
2859
2860	/*
2861	* Guest CR3.
2862	*/
2863	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2864	{
2865	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2866	if (pVM->hm.s.fNestedPaging)
2867	{
2868	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2869
2870	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2871	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2872	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2873	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2874
2875	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2876	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2877	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2878
2879	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2880	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2881	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2882	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2883
2884	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2885	AssertRCReturn(rc, rc);
2886	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2887
2888	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2889	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2890	{
2891	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2892	if (CPUMIsGuestInPAEModeEx(pCtx))
2893	{
2894	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2895	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2896	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2897	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2898	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2899	AssertRCReturn(rc, rc);
2900	}
2901
2902	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2903	have Unrestricted Execution to handle the guest when it's not using paging. */
2904	GCPhysGuestCR3 = pCtx->cr3;
2905	}
2906	else
2907	{
2908	/*
2909	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2910	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2911	* EPT takes care of translating it to host-physical addresses.
2912	*/
2913	RTGCPHYS GCPhys;
2914	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2915	Assert(PDMVmmDevHeapIsEnabled(pVM));
2916
2917	/* We obtain it here every time as the guest could have relocated this PCI region. */
2918	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2919	AssertRCReturn(rc, rc);
2920
2921	GCPhysGuestCR3 = GCPhys;
2922	}
2923	}
2924	else
2925	{
2926	/* Non-nested paging case, just use the hypervisor's CR3. */
2927	GCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2928	}
2929
2930	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv\n", GCPhysGuestCR3));
2931	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2932	AssertRCReturn(rc, rc);
2933
2934	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2935	}
2936
2937	/*
2938	* Guest CR4.
2939	*/
2940	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2941	{
2942	Assert(!(pCtx->cr4 >> 32));
2943	uint32_t u32GuestCR4 = pCtx->cr4;
2944
2945	/* The guest's view of its CR4 is unblemished. */
2946	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2947	AssertRCReturn(rc, rc);
2948	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2949
2950	/* Setup VT-x's view of the guest CR4. */
2951	/*
2952	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2953	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2954	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2955	*/
2956	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2957	{
2958	Assert(pVM->hm.s.vmx.pRealModeTSS);
2959	Assert(PDMVmmDevHeapIsEnabled(pVM));
2960	u32GuestCR4 &= ~X86_CR4_VME;
2961	}
2962
2963	if (pVM->hm.s.fNestedPaging)
2964	{
2965	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2966	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2967	{
2968	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2969	u32GuestCR4 \|= X86_CR4_PSE;
2970	/* Our identity mapping is a 32 bits page directory. */
2971	u32GuestCR4 &= ~X86_CR4_PAE;
2972	}
2973	/* else use guest CR4.*/
2974	}
2975	else
2976	{
2977	/*
2978	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2979	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2980	*/
2981	switch (pVCpu->hm.s.enmShadowMode)
2982	{
2983	case PGMMODE_REAL: /* Real-mode. */
2984	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2985	case PGMMODE_32_BIT: /* 32-bit paging. */
2986	{
2987	u32GuestCR4 &= ~X86_CR4_PAE;
2988	break;
2989	}
2990
2991	case PGMMODE_PAE: /* PAE paging. */
2992	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2993	{
2994	u32GuestCR4 \|= X86_CR4_PAE;
2995	break;
2996	}
2997
2998	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2999	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3000	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3001	break;
3002	#endif
3003	default:
3004	AssertFailed();
3005	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3006	}
3007	}
3008
3009	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3010	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3011	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3012	u32GuestCR4 \|= uSetCR4;
3013	u32GuestCR4 &= uZapCR4;
3014
3015	/* Write VT-x's view of the guest CR4 into the VMCS. */
3016	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3017	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3018
3019	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3020	uint64_t u64CR4Mask = 0;
3021	u64CR4Mask = X86_CR4_VME
3022	\| X86_CR4_PAE
3023	\| X86_CR4_PGE
3024	\| X86_CR4_PSE
3025	\| X86_CR4_VMXE;
3026	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
3027	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
3028	AssertRCReturn(rc, rc);
3029
3030	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3031	}
3032	return rc;
3033	}
3034
3035
3036	/**
3037	* Loads the guest debug registers into the guest-state area in the VMCS.
3038	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3039	*
3040	* @returns VBox status code.
3041	* @param pVCpu Pointer to the VMCPU.
3042	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3043	* out-of-sync. Make sure to update the required fields
3044	* before using them.
3045	*
3046	* @remarks No-long-jump zone!!!
3047	*/
3048	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3049	{
3050	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3051	return VINF_SUCCESS;
3052
3053	#ifdef VBOX_STRICT
3054	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3055	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3056	{
3057	Assert((pMixedCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3058	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3059	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3060	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3061	}
3062	#endif
3063
3064	int rc = VERR_INTERNAL_ERROR_5;
3065	PVM pVM = pVCpu->CTX_SUFF(pVM);
3066	bool fInterceptDB = false;
3067	bool fInterceptMovDRx = false;
3068	if (DBGFIsStepping(pVCpu))
3069	{
3070	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3071	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3072	{
3073	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3074	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3075	AssertRCReturn(rc, rc);
3076	Assert(fInterceptDB == false);
3077	}
3078	else
3079	fInterceptDB = true;
3080	}
3081
3082	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3083	{
3084	if (!CPUMIsHyperDebugStateActive(pVCpu))
3085	{
3086	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3087	AssertRC(rc);
3088	}
3089	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3090	fInterceptMovDRx = true;
3091	}
3092	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3093	{
3094	if (!CPUMIsGuestDebugStateActive(pVCpu))
3095	{
3096	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3097	AssertRC(rc);
3098	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3099	}
3100	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3101	Assert(fInterceptMovDRx == false);
3102	}
3103	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3104	{
3105	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3106	fInterceptMovDRx = true;
3107	}
3108
3109	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3110	if (fInterceptDB)
3111	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3112	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3113	{
3114	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3115	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3116	#endif
3117	}
3118
3119	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3120	if (fInterceptMovDRx)
3121	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3122	else
3123	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3124
3125	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3126	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3127
3128	/* The guest's view of its DR7 is unblemished. */
3129	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
3130
3131	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3132	return rc;
3133	}
3134
3135
3136	#ifdef VBOX_STRICT
3137	/**
3138	* Strict function to validate segment registers.
3139	*/
3140	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3141	{
3142	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3143	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3144	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3145	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3146	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3147	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3148	{
3149	/* Protected mode checks */
3150	/* CS */
3151	Assert(pCtx->cs.Attr.n.u1Present);
3152	Assert(!(pCtx->cs.Attr.u & 0xf00));
3153	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3154	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3155	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3156	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3157	\|\| (pCtx->cs.Attr.n.u1Granularity));
3158	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != HMVMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3159	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3160	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3161	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3162	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3163	else
3164	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3165	/* SS */
3166	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != HMVMX_SEL_UNUSABLE)
3167	{
3168	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3169	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3170	Assert(pCtx->ss.Attr.n.u1Present);
3171	Assert(!(pCtx->ss.Attr.u & 0xf00));
3172	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3173	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3174	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3175	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3176	\|\| (pCtx->ss.Attr.n.u1Granularity));
3177	}
3178	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3179	/* CR0 might not be up-to-date here always, hence disabled. */
3180	#if 0
3181	if (!pCtx->cr0 & X86_CR0_PE)
3182	Assert(!pCtx->ss.Attr.n.u2Dpl);
3183	#endif
3184	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3185	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != HMVMX_SEL_UNUSABLE)
3186	{
3187	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3188	Assert(pCtx->ds.Attr.n.u1Present);
3189	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3190	Assert(!(pCtx->ds.Attr.u & 0xf00));
3191	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3192	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3193	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3194	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3195	\|\| (pCtx->ds.Attr.n.u1Granularity));
3196	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3197	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3198	}
3199	if (pCtx->es.Attr.u && pCtx->es.Attr.u != HMVMX_SEL_UNUSABLE)
3200	{
3201	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3202	Assert(pCtx->es.Attr.n.u1Present);
3203	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3204	Assert(!(pCtx->es.Attr.u & 0xf00));
3205	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3206	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3207	\|\| !(pCtx->es.Attr.n.u1Granularity));
3208	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3209	\|\| (pCtx->es.Attr.n.u1Granularity));
3210	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3211	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3212	}
3213	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != HMVMX_SEL_UNUSABLE)
3214	{
3215	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3216	Assert(pCtx->fs.Attr.n.u1Present);
3217	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3218	Assert(!(pCtx->fs.Attr.u & 0xf00));
3219	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3220	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3221	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3222	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3223	\|\| (pCtx->fs.Attr.n.u1Granularity));
3224	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3225	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3226	}
3227	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != HMVMX_SEL_UNUSABLE)
3228	{
3229	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3230	Assert(pCtx->gs.Attr.n.u1Present);
3231	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3232	Assert(!(pCtx->gs.Attr.u & 0xf00));
3233	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3234	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3235	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3236	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3237	\|\| (pCtx->gs.Attr.n.u1Granularity));
3238	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3239	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3240	}
3241	/* 64-bit capable CPUs. */
3242	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3243	Assert(!(pCtx->cs.u64Base >> 32));
3244	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3245	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3246	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3247	# endif
3248	}
3249	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3250	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3251	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3252	{
3253	/* Real and v86 mode checks. */
3254	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3255	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3256	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3257	{
3258	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3259	}
3260	else
3261	{
3262	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3263	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3264	}
3265
3266	/* CS */
3267	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3268	Assert(pCtx->cs.u32Limit == 0xffff);
3269	Assert(u32CSAttr == 0xf3);
3270	/* SS */
3271	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3272	Assert(pCtx->ss.u32Limit == 0xffff);
3273	Assert(u32SSAttr == 0xf3);
3274	/* DS */
3275	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3276	Assert(pCtx->ds.u32Limit == 0xffff);
3277	Assert(u32DSAttr == 0xf3);
3278	/* ES */
3279	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3280	Assert(pCtx->es.u32Limit == 0xffff);
3281	Assert(u32ESAttr == 0xf3);
3282	/* FS */
3283	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3284	Assert(pCtx->fs.u32Limit == 0xffff);
3285	Assert(u32FSAttr == 0xf3);
3286	/* GS */
3287	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3288	Assert(pCtx->gs.u32Limit == 0xffff);
3289	Assert(u32GSAttr == 0xf3);
3290	/* 64-bit capable CPUs. */
3291	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3292	Assert(!(pCtx->cs.u64Base >> 32));
3293	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3294	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3295	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3296	# endif
3297	}
3298	}
3299	#endif /* VBOX_STRICT */
3300
3301
3302	/**
3303	* Writes a guest segment register into the guest-state area in the VMCS.
3304	*
3305	* @returns VBox status code.
3306	* @param pVCpu Pointer to the VMCPU.
3307	* @param idxSel Index of the selector in the VMCS.
3308	* @param idxLimit Index of the segment limit in the VMCS.
3309	* @param idxBase Index of the segment base in the VMCS.
3310	* @param idxAccess Index of the access rights of the segment in the VMCS.
3311	* @param pSelReg Pointer to the segment selector.
3312	* @param pCtx Pointer to the guest-CPU context.
3313	*
3314	* @remarks No-long-jump zone!!!
3315	*/
3316	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3317	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3318	{
3319	int rc;
3320	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3321	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3322	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3323	AssertRCReturn(rc, rc);
3324
3325	uint32_t u32Access = pSelReg->Attr.u;
3326	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3327	{
3328	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3329	u32Access = 0xf3;
3330	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3331	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3332	}
3333	else
3334	{
3335	/*
3336	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3337	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3338	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3339	* loaded in protected-mode have their attribute as 0.
3340	*/
3341	if (!u32Access)
3342	u32Access = HMVMX_SEL_UNUSABLE;
3343	}
3344
3345	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3346	AssertMsg((u32Access == HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3347	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3348
3349	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3350	AssertRCReturn(rc, rc);
3351	return rc;
3352	}
3353
3354
3355	/**
3356	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3357	* into the guest-state area in the VMCS.
3358	*
3359	* @returns VBox status code.
3360	* @param pVM Pointer to the VM.
3361	* @param pVCPU Pointer to the VMCPU.
3362	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3363	* out-of-sync. Make sure to update the required fields
3364	* before using them.
3365	*
3366	* @remarks No-long-jump zone!!!
3367	*/
3368	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3369	{
3370	int rc = VERR_INTERNAL_ERROR_5;
3371	PVM pVM = pVCpu->CTX_SUFF(pVM);
3372
3373	/*
3374	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3375	*/
3376	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3377	{
3378	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3379	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3380	{
3381	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3382	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3383	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3384	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3385	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3386	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3387	}
3388
3389	#ifdef VBOX_WITH_REM
3390	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3391	{
3392	Assert(pVM->hm.s.vmx.pRealModeTSS);
3393	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3394	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3395	{
3396	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3397	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3398	&& enmGuestMode >= PGMMODE_PROTECTED)
3399	{
3400	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3401	in real-mode (e.g. OpenBSD 4.0) */
3402	REMFlushTBs(pVM);
3403	Log(("Load: Switch to protected mode detected!\n"));
3404	}
3405	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3406	}
3407	}
3408	#endif
3409	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3410	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3411	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3412	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3413	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3414	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3415	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3416	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3417	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3418	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3419	rc \|= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3420	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3421	AssertRCReturn(rc, rc);
3422
3423	#ifdef VBOX_STRICT
3424	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3425	#endif
3426	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3427	}
3428
3429	/*
3430	* Guest TR.
3431	*/
3432	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3433	{
3434	/*
3435	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3436	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3437	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3438	*/
3439	uint16_t u16Sel = 0;
3440	uint32_t u32Limit = 0;
3441	uint64_t u64Base = 0;
3442	uint32_t u32AccessRights = 0;
3443
3444	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3445	{
3446	u16Sel = pMixedCtx->tr.Sel;
3447	u32Limit = pMixedCtx->tr.u32Limit;
3448	u64Base = pMixedCtx->tr.u64Base;
3449	u32AccessRights = pMixedCtx->tr.Attr.u;
3450	}
3451	else
3452	{
3453	Assert(pVM->hm.s.vmx.pRealModeTSS);
3454	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3455
3456	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3457	RTGCPHYS GCPhys;
3458	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3459	AssertRCReturn(rc, rc);
3460
3461	X86DESCATTR DescAttr;
3462	DescAttr.u = 0;
3463	DescAttr.n.u1Present = 1;
3464	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3465
3466	u16Sel = 0;
3467	u32Limit = HM_VTX_TSS_SIZE;
3468	u64Base = GCPhys; /* in real-mode phys = virt. */
3469	u32AccessRights = DescAttr.u;
3470	}
3471
3472	/* Validate. */
3473	Assert(!(u16Sel & RT_BIT(2)));
3474	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3475	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3476	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3477	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3478	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3479	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3480	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3481	Assert( (u32Limit & 0xfff) == 0xfff
3482	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3483	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3484	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3485
3486	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3487	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3488	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3489	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3490	AssertRCReturn(rc, rc);
3491
3492	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3493	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3494	}
3495
3496	/*
3497	* Guest GDTR.
3498	*/
3499	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3500	{
3501	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
3502	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
3503	AssertRCReturn(rc, rc);
3504
3505	Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3506	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3507	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3508	}
3509
3510	/*
3511	* Guest LDTR.
3512	*/
3513	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3514	{
3515	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3516	uint32_t u32Access = 0;
3517	if (!pMixedCtx->ldtr.Attr.u)
3518	u32Access = HMVMX_SEL_UNUSABLE;
3519	else
3520	u32Access = pMixedCtx->ldtr.Attr.u;
3521
3522	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel);
3523	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
3524	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
3525	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3526	AssertRCReturn(rc, rc);
3527
3528	/* Validate. */
3529	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3530	{
3531	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3532	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3533	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3534	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3535	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3536	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3537	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3538	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3539	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3540	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3541	}
3542
3543	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3544	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3545	}
3546
3547	/*
3548	* Guest IDTR.
3549	*/
3550	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3551	{
3552	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
3553	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
3554	AssertRCReturn(rc, rc);
3555
3556	Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3557	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3558	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3559	}
3560
3561	return VINF_SUCCESS;
3562	}
3563
3564
3565	/**
3566	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3567	* areas. These MSRs will automatically be loaded to the host CPU on every
3568	* successful VM entry and stored from the host CPU on every successful VM exit.
3569	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3570	*
3571	* @returns VBox status code.
3572	* @param pVCpu Pointer to the VMCPU.
3573	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3574	* out-of-sync. Make sure to update the required fields
3575	* before using them.
3576	*
3577	* @remarks No-long-jump zone!!!
3578	*/
3579	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3580	{
3581	AssertPtr(pVCpu);
3582	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3583
3584	/*
3585	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3586	*/
3587	int rc = VINF_SUCCESS;
3588	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3589	{
3590	PVM pVM = pVCpu->CTX_SUFF(pVM);
3591	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3592	uint32_t cGuestMsrs = 0;
3593
3594	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3595	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3596	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3597	if (fSupportsNX \|\| fSupportsLongMode)
3598	{
3599	/** @todo support save IA32_EFER, i.e.
3600	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3601	* guest EFER need not be part of the VM-entry MSR-load area. */
3602	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3603	pGuestMsr->u32Reserved = 0;
3604	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3605	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3606	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3607	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3608	pGuestMsr++; cGuestMsrs++;
3609	if (fSupportsLongMode)
3610	{
3611	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3612	pGuestMsr->u32Reserved = 0;
3613	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3614	pGuestMsr++; cGuestMsrs++;
3615	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3616	pGuestMsr->u32Reserved = 0;
3617	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3618	pGuestMsr++; cGuestMsrs++;
3619	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3620	pGuestMsr->u32Reserved = 0;
3621	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3622	pGuestMsr++; cGuestMsrs++;
3623	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3624	#if 0
3625	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3626	pGuestMsr->u32Reserved = 0;
3627	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3628	pGuestMsr++; cGuestMsrs++;
3629	#endif
3630	}
3631	}
3632
3633	/*
3634	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3635	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3636	*/
3637	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3638	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3639	{
3640	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3641	pGuestMsr->u32Reserved = 0;
3642	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3643	AssertRCReturn(rc, rc);
3644	pGuestMsr++; cGuestMsrs++;
3645	}
3646
3647	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3648	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3649	{
3650	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3651	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3652	}
3653
3654	/* Update the VCPU's copy of the guest MSR count. */
3655	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3656	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3657	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3658	AssertRCReturn(rc, rc);
3659
3660	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3661	}
3662
3663	/*
3664	* Guest Sysenter MSRs.
3665	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3666	* VM-exits on WRMSRs for these MSRs.
3667	*/
3668	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3669	{
3670	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
3671	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3672	}
3673	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3674	{
3675	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
3676	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3677	}
3678	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3679	{
3680	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
3681	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3682	}
3683	AssertRCReturn(rc, rc);
3684
3685	return rc;
3686	}
3687
3688
3689	/**
3690	* Loads the guest activity state into the guest-state area in the VMCS.
3691	*
3692	* @returns VBox status code.
3693	* @param pVCpu Pointer to the VMCPU.
3694	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3695	* out-of-sync. Make sure to update the required fields
3696	* before using them.
3697	*
3698	* @remarks No-long-jump zone!!!
3699	*/
3700	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3701	{
3702	/** @todo See if we can make use of other states, e.g.
3703	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3704	int rc = VINF_SUCCESS;
3705	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3706	{
3707	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3708	AssertRCReturn(rc, rc);
3709	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3710	}
3711	return rc;
3712	}
3713
3714
3715	/**
3716	* Sets up the appropriate function to run guest code.
3717	*
3718	* @returns VBox status code.
3719	* @param pVCpu Pointer to the VMCPU.
3720	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3721	* out-of-sync. Make sure to update the required fields
3722	* before using them.
3723	*
3724	* @remarks No-long-jump zone!!!
3725	*/
3726	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3727	{
3728	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3729	{
3730	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3731	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3732	#endif
3733	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3734	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3735	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3736	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3737	#else
3738	/* 64-bit host or hybrid host. */
3739	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3740	#endif
3741	}
3742	else
3743	{
3744	/* Guest is not in long mode, use the 32-bit handler. */
3745	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3746	}
3747	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3748	return VINF_SUCCESS;
3749	}
3750
3751
3752	/**
3753	* Wrapper for running the guest code in VT-x.
3754	*
3755	* @returns VBox strict status code.
3756	* @param pVM Pointer to the VM.
3757	* @param pVCpu Pointer to the VMCPU.
3758	* @param pCtx Pointer to the guest-CPU context.
3759	*
3760	* @remarks No-long-jump zone!!!
3761	*/
3762	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3763	{
3764	/*
3765	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3766	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3767	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3768	*/
3769	#ifdef VBOX_WITH_KERNEL_USING_XMM
3770	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3771	#else
3772	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3773	#endif
3774	}
3775
3776
3777	/**
3778	* Report world-switch error and dump some useful debug info.
3779	*
3780	* @param pVM Pointer to the VM.
3781	* @param pVCpu Pointer to the VMCPU.
3782	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3783	* @param pCtx Pointer to the guest-CPU context.
3784	* @param pVmxTransient Pointer to the VMX transient structure (only
3785	* exitReason updated).
3786	*/
3787	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3788	{
3789	Assert(pVM);
3790	Assert(pVCpu);
3791	Assert(pCtx);
3792	Assert(pVmxTransient);
3793	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3794
3795	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3796	switch (rcVMRun)
3797	{
3798	case VERR_VMX_INVALID_VMXON_PTR:
3799	AssertFailed();
3800	break;
3801	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3802	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3803	{
3804	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3805	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3806	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3807	AssertRC(rc);
3808
3809	#ifdef VBOX_STRICT
3810	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3811	pVmxTransient->uExitReason));
3812	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3813	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3814	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3815	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3816	else
3817	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3818
3819	/* VMX control bits. */
3820	uint32_t u32Val;
3821	uint64_t u64Val;
3822	RTHCUINTREG uHCReg;
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3827	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3828	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3829	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3830	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3831	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3832	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3833	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3834	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3835	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3836	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3837	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3838	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3839	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3840	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3841	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3842	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3843	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3844	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3845	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3846	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3847	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3848	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3849	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3850	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3851	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3852	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3853	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3854	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3855	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3856	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3857	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3858	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3859	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3860	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3861	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3862	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3863	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3864	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3865
3866	/* Guest bits. */
3867	RTGCUINTREG uGCReg;
3868	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3869	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3870	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3871	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3872	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3873	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3874	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3875	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3876
3877	/* Host bits. */
3878	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3879	Log(("Host CR0 %#RHr\n", uHCReg));
3880	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3881	Log(("Host CR3 %#RHr\n", uHCReg));
3882	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3883	Log(("Host CR4 %#RHr\n", uHCReg));
3884
3885	RTGDTR HostGdtr;
3886	PCX86DESCHC pDesc;
3887	ASMGetGDTR(&HostGdtr);
3888	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3889	Log(("Host CS %#08x\n", u32Val));
3890	if (u32Val < HostGdtr.cbGdt)
3891	{
3892	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3893	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3894	}
3895
3896	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3897	Log(("Host DS %#08x\n", u32Val));
3898	if (u32Val < HostGdtr.cbGdt)
3899	{
3900	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3901	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3902	}
3903
3904	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3905	Log(("Host ES %#08x\n", u32Val));
3906	if (u32Val < HostGdtr.cbGdt)
3907	{
3908	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3909	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3910	}
3911
3912	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3913	Log(("Host FS %#08x\n", u32Val));
3914	if (u32Val < HostGdtr.cbGdt)
3915	{
3916	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3917	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3918	}
3919
3920	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3921	Log(("Host GS %#08x\n", u32Val));
3922	if (u32Val < HostGdtr.cbGdt)
3923	{
3924	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3925	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3926	}
3927
3928	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3929	Log(("Host SS %#08x\n", u32Val));
3930	if (u32Val < HostGdtr.cbGdt)
3931	{
3932	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3933	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3934	}
3935
3936	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3937	Log(("Host TR %#08x\n", u32Val));
3938	if (u32Val < HostGdtr.cbGdt)
3939	{
3940	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3941	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3942	}
3943
3944	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3945	Log(("Host TR Base %#RHv\n", uHCReg));
3946	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3947	Log(("Host GDTR Base %#RHv\n", uHCReg));
3948	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3949	Log(("Host IDTR Base %#RHv\n", uHCReg));
3950	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3951	Log(("Host SYSENTER CS %#08x\n", u32Val));
3952	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3953	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3954	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3955	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3956	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3957	Log(("Host RSP %#RHv\n", uHCReg));
3958	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3959	Log(("Host RIP %#RHv\n", uHCReg));
3960	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3961	if (HMVMX_IS_64BIT_HOST_MODE())
3962	{
3963	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3964	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3965	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3966	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3967	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3968	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3969	}
3970	# endif
3971	#endif /* VBOX_STRICT */
3972	break;
3973	}
3974
3975	default:
3976	/* Impossible */
3977	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3978	break;
3979	}
3980	NOREF(pVM);
3981	}
3982
3983
3984	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3985	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3986	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3987	#endif
3988	#ifdef VBOX_STRICT
3989	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3990	{
3991	switch (idxField)
3992	{
3993	case VMX_VMCS_GUEST_RIP:
3994	case VMX_VMCS_GUEST_RSP:
3995	case VMX_VMCS_GUEST_DR7:
3996	case VMX_VMCS_GUEST_SYSENTER_EIP:
3997	case VMX_VMCS_GUEST_SYSENTER_ESP:
3998	case VMX_VMCS_GUEST_GDTR_BASE:
3999	case VMX_VMCS_GUEST_IDTR_BASE:
4000	case VMX_VMCS_GUEST_CS_BASE:
4001	case VMX_VMCS_GUEST_DS_BASE:
4002	case VMX_VMCS_GUEST_ES_BASE:
4003	case VMX_VMCS_GUEST_FS_BASE:
4004	case VMX_VMCS_GUEST_GS_BASE:
4005	case VMX_VMCS_GUEST_SS_BASE:
4006	case VMX_VMCS_GUEST_LDTR_BASE:
4007	case VMX_VMCS_GUEST_TR_BASE:
4008	case VMX_VMCS_GUEST_CR3:
4009	return true;
4010	}
4011	return false;
4012	}
4013
4014	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4015	{
4016	switch (idxField)
4017	{
4018	/* Read-only fields. */
4019	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4020	return true;
4021	}
4022	/* All readable fields should also be part of the VMCS write cache. */
4023	return hmR0VmxIsValidWriteField(idxField);
4024	}
4025	#endif /* VBOX_STRICT */
4026
4027	/**
4028	* Executes the specified handler in 64-bit mode.
4029	*
4030	* @returns VBox status code.
4031	* @param pVM Pointer to the VM.
4032	* @param pVCpu Pointer to the VMCPU.
4033	* @param pCtx Pointer to the guest CPU context.
4034	* @param pfnHandler Pointer to the RC handler function.
4035	* @param cbParam Number of parameters.
4036	* @param paParam Array of 32-bit parameters.
4037	*/
4038	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4039	uint32_t *paParam)
4040	{
4041	int rc, rc2;
4042	PHMGLOBLCPUINFO pCpu;
4043	RTHCPHYS HCPhysCpuPage;
4044	RTHCUINTREG uOldEFlags;
4045
4046	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4047	Assert(pfnHandler);
4048	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4049	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4050
4051	#ifdef VBOX_STRICT
4052	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4053	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4054
4055	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4056	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4057	#endif
4058
4059	/* Disable interrupts. */
4060	uOldEFlags = ASMIntDisableFlags();
4061
4062	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4063	RTCPUID idHostCpu = RTMpCpuId();
4064	CPUMR0SetLApic(pVM, idHostCpu);
4065	#endif
4066
4067	pCpu = HMR0GetCurrentCpu();
4068	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4069
4070	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4071	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4072
4073	/* Leave VMX Root Mode. */
4074	VMXDisable();
4075
4076	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4077
4078	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4079	CPUMSetHyperEIP(pVCpu, pfnHandler);
4080	for (int i = (int)cbParam - 1; i >= 0; i--)
4081	CPUMPushHyper(pVCpu, paParam[i]);
4082
4083	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4084
4085	/* Call the switcher. */
4086	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4087	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4088
4089	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4090	/* Make sure the VMX instructions don't cause #UD faults. */
4091	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4092
4093	/* Re-enter VMX Root Mode */
4094	rc2 = VMXEnable(HCPhysCpuPage);
4095	if (RT_FAILURE(rc2))
4096	{
4097	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4098	ASMSetFlags(uOldEFlags);
4099	return rc2;
4100	}
4101
4102	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4103	AssertRC(rc2);
4104	Assert(!(ASMGetFlags() & X86_EFL_IF));
4105	ASMSetFlags(uOldEFlags);
4106	return rc;
4107	}
4108
4109
4110	/**
4111	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4112	* supporting 64-bit guests.
4113	*
4114	* @returns VBox status code.
4115	* @param fResume Whether to VMLAUNCH or VMRESUME.
4116	* @param pCtx Pointer to the guest-CPU context.
4117	* @param pCache Pointer to the VMCS cache.
4118	* @param pVM Pointer to the VM.
4119	* @param pVCpu Pointer to the VMCPU.
4120	*/
4121	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4122	{
4123	uint32_t aParam[6];
4124	PHMGLOBLCPUINFO pCpu = NULL;
4125	RTHCPHYS HCPhysCpuPage = 0;
4126	int rc = VERR_INTERNAL_ERROR_5;
4127
4128	pCpu = HMR0GetCurrentCpu();
4129	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4130
4131	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4132	pCache->uPos = 1;
4133	pCache->interPD = PGMGetInterPaeCR3(pVM);
4134	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4135	#endif
4136
4137	#ifdef VBOX_STRICT
4138	pCache->TestIn.HCPhysCpuPage = 0;
4139	pCache->TestIn.HCPhysVmcs = 0;
4140	pCache->TestIn.pCache = 0;
4141	pCache->TestOut.HCPhysVmcs = 0;
4142	pCache->TestOut.pCache = 0;
4143	pCache->TestOut.pCtx = 0;
4144	pCache->TestOut.eflags = 0;
4145	#endif
4146
4147	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4148	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4149	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4150	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4151	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4152	aParam[5] = 0;
4153
4154	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4155	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4156	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4157	#endif
4158	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4159
4160	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4161	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4162	Assert(pCtx->dr[4] == 10);
4163	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4164	#endif
4165
4166	#ifdef VBOX_STRICT
4167	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4168	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4169	pVCpu->hm.s.vmx.HCPhysVmcs));
4170	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4171	pCache->TestOut.HCPhysVmcs));
4172	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4173	pCache->TestOut.pCache));
4174	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4175	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4176	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4177	pCache->TestOut.pCtx));
4178	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4179	#endif
4180	return rc;
4181	}
4182
4183
4184	/**
4185	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4186	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4187	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4188	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4189	*
4190	* @returns VBox status code.
4191	* @param pVM Pointer to the VM.
4192	* @param pVCpu Pointer to the VMCPU.
4193	*/
4194	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4195	{
4196	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4197	{ \
4198	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4199	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4200	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4201	++cReadFields; \
4202	}
4203
4204	AssertPtr(pVM);
4205	AssertPtr(pVCpu);
4206	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4207	uint32_t cReadFields = 0;
4208
4209	/* Guest-natural selector base fields */
4210	#if 0
4211	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4213	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4214	#endif
4215	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4216	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4217	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4218	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4219	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4220	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4221	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4222	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4223	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4224	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4225	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4226	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4227	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4228	#if 0
4229	/* Unused natural width guest-state fields. */
4230	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4231	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4232	#endif
4233	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4234	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4235
4236	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4237	#if 0
4238	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4239	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4240	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4241	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4242	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4243	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4244	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4245	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4246	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4247	#endif
4248
4249	/* Natural width guest-state fields. */
4250	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4251	#if 0
4252	/* Currently unused field. */
4253	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4254	#endif
4255
4256	if (pVM->hm.s.fNestedPaging)
4257	{
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4259	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4260	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4261	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4262	}
4263	else
4264	{
4265	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4266	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4267	}
4268
4269	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4270	return VINF_SUCCESS;
4271	}
4272
4273
4274	/**
4275	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4276	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4277	* darwin, running 64-bit guests).
4278	*
4279	* @returns VBox status code.
4280	* @param pVCpu Pointer to the VMCPU.
4281	* @param idxField The VMCS field encoding.
4282	* @param u64Val 16, 32 or 64 bits value.
4283	*/
4284	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4285	{
4286	int rc;
4287	switch (idxField)
4288	{
4289	/*
4290	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4291	*/
4292	/* 64-bit Control fields. */
4293	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4294	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4295	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4296	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4297	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4298	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4299	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4300	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4301	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4302	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4303	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4304	case VMX_VMCS64_CTRL_EPTP_FULL:
4305	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4306	/* 64-bit Guest-state fields. */
4307	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4308	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4309	case VMX_VMCS64_GUEST_PAT_FULL:
4310	case VMX_VMCS64_GUEST_EFER_FULL:
4311	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4312	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4313	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4314	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4315	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4316	/* 64-bit Host-state fields. */
4317	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4318	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4319	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4320	{
4321	rc = VMXWriteVmcs32(idxField, u64Val);
4322	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4323	break;
4324	}
4325
4326	/*
4327	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4328	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4329	*/
4330	/* Natural-width Guest-state fields. */
4331	case VMX_VMCS_GUEST_CR3:
4332	case VMX_VMCS_GUEST_ES_BASE:
4333	case VMX_VMCS_GUEST_CS_BASE:
4334	case VMX_VMCS_GUEST_SS_BASE:
4335	case VMX_VMCS_GUEST_DS_BASE:
4336	case VMX_VMCS_GUEST_FS_BASE:
4337	case VMX_VMCS_GUEST_GS_BASE:
4338	case VMX_VMCS_GUEST_LDTR_BASE:
4339	case VMX_VMCS_GUEST_TR_BASE:
4340	case VMX_VMCS_GUEST_GDTR_BASE:
4341	case VMX_VMCS_GUEST_IDTR_BASE:
4342	case VMX_VMCS_GUEST_DR7:
4343	case VMX_VMCS_GUEST_RSP:
4344	case VMX_VMCS_GUEST_RIP:
4345	case VMX_VMCS_GUEST_SYSENTER_ESP:
4346	case VMX_VMCS_GUEST_SYSENTER_EIP:
4347	{
4348	if (!(u64Val >> 32))
4349	{
4350	/* If this field is 64-bit, VT-x will zero out the top bits. */
4351	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4352	}
4353	else
4354	{
4355	/* Assert that only the 32->64 switcher case should ever come here. */
4356	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4357	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4358	}
4359	break;
4360	}
4361
4362	default:
4363	{
4364	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4365	rc = VERR_INVALID_PARAMETER;
4366	break;
4367	}
4368	}
4369	AssertRCReturn(rc, rc);
4370	return rc;
4371	}
4372
4373
4374	/**
4375	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4376	* hosts (except darwin) for 64-bit guests.
4377	*
4378	* @param pVCpu Pointer to the VMCPU.
4379	* @param idxField The VMCS field encoding.
4380	* @param u64Val 16, 32 or 64 bits value.
4381	*/
4382	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4383	{
4384	AssertPtr(pVCpu);
4385	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4386
4387	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4388	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4389
4390	/* Make sure there are no duplicates. */
4391	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4392	{
4393	if (pCache->Write.aField[i] == idxField)
4394	{
4395	pCache->Write.aFieldVal[i] = u64Val;
4396	return VINF_SUCCESS;
4397	}
4398	}
4399
4400	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4401	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4402	pCache->Write.cValidEntries++;
4403	return VINF_SUCCESS;
4404	}
4405
4406	/* Enable later when the assembly code uses these as callbacks. */
4407	#if 0
4408	/*
4409	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4410	*
4411	* @param pVCpu Pointer to the VMCPU.
4412	* @param pCache Pointer to the VMCS cache.
4413	*
4414	* @remarks No-long-jump zone!!!
4415	*/
4416	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4417	{
4418	AssertPtr(pCache);
4419	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4420	{
4421	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4422	AssertRC(rc);
4423	}
4424	pCache->Write.cValidEntries = 0;
4425	}
4426
4427
4428	/**
4429	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4430	*
4431	* @param pVCpu Pointer to the VMCPU.
4432	* @param pCache Pointer to the VMCS cache.
4433	*
4434	* @remarks No-long-jump zone!!!
4435	*/
4436	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4437	{
4438	AssertPtr(pCache);
4439	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4440	{
4441	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4442	AssertRC(rc);
4443	}
4444	}
4445	#endif
4446	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4447
4448
4449	/**
4450	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4451	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4452	* timer.
4453	*
4454	* @returns VBox status code.
4455	* @param pVCpu Pointer to the VMCPU.
4456	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4457	* out-of-sync. Make sure to update the required fields
4458	* before using them.
4459	* @remarks No-long-jump zone!!!
4460	*/
4461	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4462	{
4463	int rc = VERR_INTERNAL_ERROR_5;
4464	bool fOffsettedTsc = false;
4465	PVM pVM = pVCpu->CTX_SUFF(pVM);
4466	if (pVM->hm.s.vmx.fUsePreemptTimer)
4467	{
4468	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4469
4470	/* Make sure the returned values have sane upper and lower boundaries. */
4471	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4472	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4473	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4474	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4475
4476	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4477	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4478	}
4479	else
4480	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4481
4482	if (fOffsettedTsc)
4483	{
4484	uint64_t u64CurTSC = ASMReadTSC();
4485	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4486	{
4487	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4488	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4489
4490	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4491	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4492	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4493	}
4494	else
4495	{
4496	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4497	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4498	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4499	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4500	}
4501	}
4502	else
4503	{
4504	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4505	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4506	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4507	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4508	}
4509	}
4510
4511
4512	/**
4513	* Determines if an exception is a contributory exception. Contributory
4514	* exceptions are ones which can cause double-faults. Page-fault is
4515	* intentionally not included here as it's a conditional contributory exception.
4516	*
4517	* @returns true if the exception is contributory, false otherwise.
4518	* @param uVector The exception vector.
4519	*/
4520	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4521	{
4522	switch (uVector)
4523	{
4524	case X86_XCPT_GP:
4525	case X86_XCPT_SS:
4526	case X86_XCPT_NP:
4527	case X86_XCPT_TS:
4528	case X86_XCPT_DE:
4529	return true;
4530	default:
4531	break;
4532	}
4533	return false;
4534	}
4535
4536
4537	/**
4538	* Sets an event as a pending event to be injected into the guest.
4539	*
4540	* @param pVCpu Pointer to the VMCPU.
4541	* @param u32IntrInfo The VM-entry interruption-information field.
4542	* @param cbInstr The VM-entry instruction length in bytes (for software
4543	* interrupts, exceptions and privileged software
4544	* exceptions).
4545	* @param u32ErrCode The VM-entry exception error code.
4546	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4547	* page-fault.
4548	*/
4549	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4550	RTGCUINTPTR GCPtrFaultAddress)
4551	{
4552	Assert(!pVCpu->hm.s.Event.fPending);
4553	pVCpu->hm.s.Event.fPending = true;
4554	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4555	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4556	pVCpu->hm.s.Event.cbInstr = cbInstr;
4557	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4558	}
4559
4560
4561	/**
4562	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4563	*
4564	* @param pVCpu Pointer to the VMCPU.
4565	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4566	* out-of-sync. Make sure to update the required fields
4567	* before using them.
4568	*/
4569	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4570	{
4571	/* Inject the double-fault. */
4572	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4573	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4574	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4575	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4576	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4577	}
4578
4579
4580	/**
4581	* Handle a condition that occurred while delivering an event through the guest
4582	* IDT.
4583	*
4584	* @returns VBox status code (informational error codes included).
4585	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4586	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4587	* continue execution of the guest which will delivery the #DF.
4588	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4589	*
4590	* @param pVCpu Pointer to the VMCPU.
4591	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4592	* out-of-sync. Make sure to update the required fields
4593	* before using them.
4594	* @param pVmxTransient Pointer to the VMX transient structure.
4595	*
4596	* @remarks No-long-jump zone!!!
4597	*/
4598	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4599	{
4600	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4601	AssertRC(rc);
4602	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4603	{
4604	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4605	AssertRCReturn(rc, rc);
4606
4607	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4608	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4609	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4610
4611	typedef enum
4612	{
4613	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4614	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4615	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4616	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4617	} VMXREFLECTXCPT;
4618
4619	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4620	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4621	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4622	{
4623	enmReflect = VMXREFLECTXCPT_XCPT;
4624	#if 0
4625	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4626	&& uExitVector == X86_XCPT_PF)
4627	{
4628	Log(("IDT: Contributory #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4629	}
4630	else
4631	#endif
4632	if ( uExitVector == X86_XCPT_PF
4633	&& uIdtVector == X86_XCPT_PF)
4634	{
4635	pVmxTransient->fVectoringPF = true;
4636	Log(("IDT: Vectoring #PF uCR2=%#RGv\n", pMixedCtx->cr2));
4637	}
4638	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4639	&& hmR0VmxIsContributoryXcpt(uExitVector)
4640	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4641	\|\| uIdtVector == X86_XCPT_PF))
4642	{
4643	enmReflect = VMXREFLECTXCPT_DF;
4644	}
4645	else if (uIdtVector == X86_XCPT_DF)
4646	enmReflect = VMXREFLECTXCPT_TF;
4647	}
4648	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4649	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4650	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4651	{
4652	/*
4653	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4654	* (whatever they are) as they reoccur when restarting the instruction.
4655	*/
4656	enmReflect = VMXREFLECTXCPT_XCPT;
4657	}
4658
4659	switch (enmReflect)
4660	{
4661	case VMXREFLECTXCPT_XCPT:
4662	{
4663	uint32_t u32ErrCode = 0;
4664	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4665	{
4666	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4667	AssertRCReturn(rc, rc);
4668	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4669	}
4670
4671	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4672	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4673	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4674	rc = VINF_SUCCESS;
4675	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4676	break;
4677	}
4678
4679	case VMXREFLECTXCPT_DF:
4680	{
4681	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4682	rc = VINF_VMX_DOUBLE_FAULT;
4683	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4684	uExitVector));
4685	break;
4686	}
4687
4688	case VMXREFLECTXCPT_TF:
4689	{
4690	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4691	rc = VINF_EM_RESET;
4692	break;
4693	}
4694
4695	default:
4696	Assert(rc == VINF_SUCCESS);
4697	break;
4698	}
4699	}
4700	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4701	return rc;
4702	}
4703
4704
4705	/**
4706	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4707	*
4708	* @returns VBox status code.
4709	* @param pVCpu Pointer to the VMCPU.
4710	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4711	* out-of-sync. Make sure to update the required fields
4712	* before using them.
4713	*
4714	* @remarks No-long-jump zone!!!
4715	*/
4716	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4717	{
4718	int rc = VINF_SUCCESS;
4719	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4720	{
4721	uint32_t uVal = 0;
4722	uint32_t uShadow = 0;
4723	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4724	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4725	AssertRCReturn(rc, rc);
4726	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4727	CPUMSetGuestCR0(pVCpu, uVal);
4728	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4729	}
4730	return rc;
4731	}
4732
4733
4734	/**
4735	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4736	*
4737	* @returns VBox status code.
4738	* @param pVCpu Pointer to the VMCPU.
4739	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4740	* out-of-sync. Make sure to update the required fields
4741	* before using them.
4742	*
4743	* @remarks No-long-jump zone!!!
4744	*/
4745	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4746	{
4747	int rc = VINF_SUCCESS;
4748	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4749	{
4750	uint32_t uVal = 0;
4751	uint32_t uShadow = 0;
4752	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4753	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4754	AssertRCReturn(rc, rc);
4755	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4756	CPUMSetGuestCR4(pVCpu, uVal);
4757	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4758	}
4759	return rc;
4760	}
4761
4762
4763	/**
4764	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4765	*
4766	* @returns VBox status code.
4767	* @param pVCpu Pointer to the VMCPU.
4768	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4769	* out-of-sync. Make sure to update the required fields
4770	* before using them.
4771	*
4772	* @remarks No-long-jump zone!!!
4773	*/
4774	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4775	{
4776	int rc = VINF_SUCCESS;
4777	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4778	{
4779	RTGCUINTREG uVal = 0;
4780	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4781	AssertRCReturn(rc, rc);
4782	pMixedCtx->rip = uVal;
4783	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4784	}
4785	return rc;
4786	}
4787
4788
4789	/**
4790	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4791	*
4792	* @returns VBox status code.
4793	* @param pVCpu Pointer to the VMCPU.
4794	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4795	* out-of-sync. Make sure to update the required fields
4796	* before using them.
4797	*
4798	* @remarks No-long-jump zone!!!
4799	*/
4800	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4801	{
4802	int rc = VINF_SUCCESS;
4803	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4804	{
4805	RTGCUINTREG uVal = 0;
4806	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4807	AssertRCReturn(rc, rc);
4808	pMixedCtx->rsp = uVal;
4809	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4810	}
4811	return rc;
4812	}
4813
4814
4815	/**
4816	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4817	*
4818	* @returns VBox status code.
4819	* @param pVCpu Pointer to the VMCPU.
4820	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4821	* out-of-sync. Make sure to update the required fields
4822	* before using them.
4823	*
4824	* @remarks No-long-jump zone!!!
4825	*/
4826	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4827	{
4828	int rc = VINF_SUCCESS;
4829	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4830	{
4831	uint32_t uVal = 0;
4832	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4833	AssertRCReturn(rc, rc);
4834	pMixedCtx->eflags.u32 = uVal;
4835
4836	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4837	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4838	{
4839	PVM pVM = pVCpu->CTX_SUFF(pVM);
4840	Assert(pVM->hm.s.vmx.pRealModeTSS);
4841	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4842	pMixedCtx->eflags.Bits.u1VM = 0;
4843	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4844	}
4845
4846	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4847	}
4848	return rc;
4849	}
4850
4851
4852	/**
4853	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4854	* guest-CPU context.
4855	*/
4856	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4857	{
4858	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4859	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4860	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4861	return rc;
4862	}
4863
4864
4865	/**
4866	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4867	* from the guest-state area in the VMCS.
4868	*
4869	* @param pVCpu Pointer to the VMCPU.
4870	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4871	* out-of-sync. Make sure to update the required fields
4872	* before using them.
4873	*
4874	* @remarks No-long-jump zone!!!
4875	*/
4876	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4877	{
4878	uint32_t uIntrState = 0;
4879	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4880	AssertRC(rc);
4881
4882	if (!uIntrState)
4883	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4884	else
4885	{
4886	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4887	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4888	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4889	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4890	AssertRC(rc);
4891	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4892	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4893	}
4894	}
4895
4896
4897	/**
4898	* Saves the guest's activity state.
4899	*
4900	* @returns VBox status code.
4901	* @param pVCpu Pointer to the VMCPU.
4902	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4903	* out-of-sync. Make sure to update the required fields
4904	* before using them.
4905	*
4906	* @remarks No-long-jump zone!!!
4907	*/
4908	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4909	{
4910	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4911	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4912	return VINF_SUCCESS;
4913	}
4914
4915
4916	/**
4917	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4918	* the current VMCS into the guest-CPU context.
4919	*
4920	* @returns VBox status code.
4921	* @param pVCpu Pointer to the VMCPU.
4922	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4923	* out-of-sync. Make sure to update the required fields
4924	* before using them.
4925	*
4926	* @remarks No-long-jump zone!!!
4927	*/
4928	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4929	{
4930	int rc = VINF_SUCCESS;
4931	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4932	{
4933	uint32_t u32Val = 0;
4934	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4935	pMixedCtx->SysEnter.cs = u32Val;
4936	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4937	}
4938
4939	RTGCUINTREG uGCVal = 0;
4940	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4941	{
4942	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4943	pMixedCtx->SysEnter.eip = uGCVal;
4944	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4945	}
4946	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4947	{
4948	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4949	pMixedCtx->SysEnter.esp = uGCVal;
4950	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4951	}
4952	return rc;
4953	}
4954
4955
4956	/**
4957	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4958	* context.
4959	*
4960	* @returns VBox status code.
4961	* @param pVCpu Pointer to the VMCPU.
4962	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4963	* out-of-sync. Make sure to update the required fields
4964	* before using them.
4965	*
4966	* @remarks No-long-jump zone!!!
4967	*/
4968	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4969	{
4970	int rc = VINF_SUCCESS;
4971	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
4972	{
4973	RTGCUINTREG uVal = 0;
4974	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4975	pMixedCtx->fs.u64Base = uVal;
4976	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
4977	}
4978	return rc;
4979	}
4980
4981
4982	/**
4983	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4984	* context.
4985	*
4986	* @returns VBox status code.
4987	* @param pVCpu Pointer to the VMCPU.
4988	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4989	* out-of-sync. Make sure to update the required fields
4990	* before using them.
4991	*
4992	* @remarks No-long-jump zone!!!
4993	*/
4994	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4995	{
4996	int rc = VINF_SUCCESS;
4997	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
4998	{
4999	RTGCUINTREG uVal = 0;
5000	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5001	pMixedCtx->gs.u64Base = uVal;
5002	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5003	}
5004	return rc;
5005	}
5006
5007
5008	/**
5009	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5010	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5011	*
5012	* @returns VBox status code.
5013	* @param pVCpu Pointer to the VMCPU.
5014	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5015	* out-of-sync. Make sure to update the required fields
5016	* before using them.
5017	*
5018	* @remarks No-long-jump zone!!!
5019	*/
5020	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5021	{
5022	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5023	return VINF_SUCCESS;
5024
5025	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5026	{
5027	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5028	pMsr += i;
5029	switch (pMsr->u32IndexMSR)
5030	{
5031	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5032	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5033	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5034	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5035	#if 0
5036	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5037	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5038	#endif
5039	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5040	default:
5041	{
5042	AssertFailed();
5043	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5044	}
5045	}
5046	}
5047	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5048	return VINF_SUCCESS;
5049	}
5050
5051
5052	/**
5053	* Saves the guest control registers from the current VMCS into the guest-CPU
5054	* context.
5055	*
5056	* @returns VBox status code.
5057	* @param pVCpu Pointer to the VMCPU.
5058	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5059	* out-of-sync. Make sure to update the required fields
5060	* before using them.
5061	*
5062	* @remarks No-long-jump zone!!!
5063	*/
5064	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5065	{
5066	/* Guest CR0. Guest FPU. */
5067	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5068
5069	/* Guest CR4. */
5070	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5071	AssertRCReturn(rc, rc);
5072
5073	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5074	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5075	{
5076	PVM pVM = pVCpu->CTX_SUFF(pVM);
5077	if ( pVM->hm.s.fNestedPaging
5078	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5079	{
5080	RTGCUINTREG uVal = 0;
5081	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5082	if (pMixedCtx->cr3 != uVal)
5083	{
5084	CPUMSetGuestCR3(pVCpu, uVal);
5085	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5086	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5087	}
5088
5089	/* We require EFER to check PAE mode. */
5090	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5091
5092	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5093	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5094	{
5095	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5096	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5097	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5098	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5099	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5100	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5101	}
5102	AssertRCReturn(rc, rc);
5103	}
5104
5105	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5106	}
5107	return rc;
5108	}
5109
5110
5111	/**
5112	* Reads a guest segment register from the current VMCS into the guest-CPU
5113	* context.
5114	*
5115	* @returns VBox status code.
5116	* @param pVCpu Pointer to the VMCPU.
5117	* @param idxSel Index of the selector in the VMCS.
5118	* @param idxLimit Index of the segment limit in the VMCS.
5119	* @param idxBase Index of the segment base in the VMCS.
5120	* @param idxAccess Index of the access rights of the segment in the VMCS.
5121	* @param pSelReg Pointer to the segment selector.
5122	*
5123	* @remarks No-long-jump zone!!!
5124	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5125	* as that takes care of whether to read from the VMCS cache or not.
5126	*/
5127	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5128	PCPUMSELREG pSelReg)
5129	{
5130	uint32_t u32Val = 0;
5131	int rc = VMXReadVmcs32(idxSel, &u32Val);
5132	pSelReg->Sel = (uint16_t)u32Val;
5133	pSelReg->ValidSel = (uint16_t)u32Val;
5134	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5135
5136	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5137	pSelReg->u32Limit = u32Val;
5138
5139	RTGCUINTREG uGCVal = 0;
5140	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5141	pSelReg->u64Base = uGCVal;
5142
5143	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5144	pSelReg->Attr.u = u32Val;
5145	AssertRCReturn(rc, rc);
5146
5147	/*
5148	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5149	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5150	*/
5151	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5152	{
5153	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5154	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5155	}
5156	return rc;
5157	}
5158
5159
5160	/**
5161	* Saves the guest segment registers from the current VMCS into the guest-CPU
5162	* context.
5163	*
5164	* @returns VBox status code.
5165	* @param pVCpu Pointer to the VMCPU.
5166	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5167	* out-of-sync. Make sure to update the required fields
5168	* before using them.
5169	*
5170	* @remarks No-long-jump zone!!!
5171	*/
5172	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5173	{
5174	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5175	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5176	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5177	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5178	#else
5179	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5180	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5181	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5182	#endif
5183
5184	int rc = VINF_SUCCESS;
5185
5186	/* Guest segment registers. */
5187	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5188	{
5189	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5190	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5191	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5192	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5193	rc \|= VMXLOCAL_READ_SEG(ES, es);
5194	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5195	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5196	AssertRCReturn(rc, rc);
5197
5198	/* Restore segment attributes for real-on-v86 mode hack. */
5199	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5200	{
5201	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5202	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5203	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5204	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5205	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5206	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5207	}
5208	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5209	}
5210
5211	/* Guest LDTR. */
5212	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5213	{
5214	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5215	AssertRCReturn(rc, rc);
5216	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5217	}
5218
5219	/* Guest GDTR. */
5220	RTGCUINTREG uGCVal = 0;
5221	uint32_t u32Val = 0;
5222	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5223	{
5224	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5225	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5226	pMixedCtx->gdtr.pGdt = uGCVal;
5227	pMixedCtx->gdtr.cbGdt = u32Val;
5228	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5229	}
5230
5231	/* Guest IDTR. */
5232	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5233	{
5234	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5235	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5236	pMixedCtx->idtr.pIdt = uGCVal;
5237	pMixedCtx->idtr.cbIdt = u32Val;
5238	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5239	}
5240
5241	/* Guest TR. */
5242	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5243	{
5244	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5245
5246	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5247	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5248	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5249	AssertRCReturn(rc, rc);
5250	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5251	}
5252	return rc;
5253	}
5254
5255
5256	/**
5257	* Saves the guest debug registers from the current VMCS into the guest-CPU
5258	* context.
5259	*
5260	* @returns VBox status code.
5261	* @param pVCpu Pointer to the VMCPU.
5262	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5263	* out-of-sync. Make sure to update the required fields
5264	* before using them.
5265	*
5266	* @remarks No-long-jump zone!!!
5267	*/
5268	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5269	{
5270	int rc = VINF_SUCCESS;
5271	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5272	{
5273	RTGCUINTREG uVal;
5274	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5275	pMixedCtx->dr[7] = uVal;
5276
5277	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5278	}
5279	return rc;
5280	}
5281
5282
5283	/**
5284	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5285	*
5286	* @returns VBox status code.
5287	* @param pVCpu Pointer to the VMCPU.
5288	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5289	* out-of-sync. Make sure to update the required fields
5290	* before using them.
5291	*
5292	* @remarks No-long-jump zone!!!
5293	*/
5294	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5295	{
5296	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5297	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5298	return VINF_SUCCESS;
5299	}
5300
5301
5302	/**
5303	* Saves the entire guest state from the currently active VMCS into the
5304	* guest-CPU context. This essentially VMREADs all guest-data.
5305	*
5306	* @returns VBox status code.
5307	* @param pVCpu Pointer to the VMCPU.
5308	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5309	* out-of-sync. Make sure to update the required fields
5310	* before using them.
5311	*/
5312	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5313	{
5314	Assert(pVCpu);
5315	Assert(pMixedCtx);
5316
5317	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5318	return VINF_SUCCESS;
5319
5320	VMMRZCallRing3Disable(pVCpu);
5321	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5322	LogFunc(("\n"));
5323
5324	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5325	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5326
5327	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5328	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5329
5330	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5331	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5332
5333	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5334	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5335
5336	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5337	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5338
5339	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5340	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5341
5342	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5343	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5344
5345	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5346	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5347
5348	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5349	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5350
5351	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5352	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5353
5354	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5355	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5356
5357	VMMRZCallRing3Enable(pVCpu);
5358	return rc;
5359	}
5360
5361
5362	/**
5363	* Check per-VM and per-VCPU force flag actions that require us to go back to
5364	* ring-3 for one reason or another.
5365	*
5366	* @returns VBox status code (information status code included).
5367	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5368	* ring-3.
5369	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5370	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5371	* interrupts)
5372	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5373	* all EMTs to be in ring-3.
5374	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5375	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5376	* to the EM loop.
5377	*
5378	* @param pVM Pointer to the VM.
5379	* @param pVCpu Pointer to the VMCPU.
5380	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5381	* out-of-sync. Make sure to update the required fields
5382	* before using them.
5383	*/
5384	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5385	{
5386	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5387
5388	int rc = VERR_INTERNAL_ERROR_5;
5389	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5390	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5391	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5392	{
5393	/* We need the control registers now, make sure the guest-CPU context is updated. */
5394	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5395	AssertRCReturn(rc, rc);
5396
5397	/* Pending HM CR3 sync. */
5398	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5399	{
5400	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5401	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5402	}
5403	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5404	{
5405	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5406	AssertRC(rc);
5407	}
5408
5409	/* Pending PGM C3 sync. */
5410	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5411	{
5412	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5413	if (rc != VINF_SUCCESS)
5414	{
5415	AssertRC(rc);
5416	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5417	return rc;
5418	}
5419	}
5420
5421	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5422	/* -XXX- what was that about single stepping? */
5423	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5424	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5425	{
5426	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5427	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5428	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5429	return rc;
5430	}
5431
5432	/* Pending VM request packets, such as hardware interrupts. */
5433	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5434	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5435	{
5436	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5437	return VINF_EM_PENDING_REQUEST;
5438	}
5439
5440	/* Pending PGM pool flushes. */
5441	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5442	{
5443	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5444	return VINF_PGM_POOL_FLUSH_PENDING;
5445	}
5446
5447	/* Pending DMA requests. */
5448	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5449	{
5450	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5451	return VINF_EM_RAW_TO_R3;
5452	}
5453	}
5454
5455	/* Paranoia. */
5456	Assert(rc != VERR_EM_INTERPRETER);
5457	return VINF_SUCCESS;
5458	}
5459
5460
5461	/**
5462	* Converts any TRPM trap into a pending VMX event. This is typically used when
5463	* entering from ring-3 (not longjmp returns).
5464	*
5465	* @param pVCpu Pointer to the VMCPU.
5466	* @param pCtx Pointer to the guest-CPU context.
5467	*/
5468	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5469	{
5470	if (!TRPMHasTrap(pVCpu))
5471	{
5472	Assert(!pVCpu->hm.s.Event.fPending);
5473	return;
5474	}
5475
5476	uint8_t uVector;
5477	TRPMEVENT enmTrpmEvent;
5478	RTGCUINT uErrCode;
5479	RTGCUINTPTR GCPtrFaultAddress;
5480	uint8_t cbInstr;
5481
5482	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5483	AssertRC(rc);
5484
5485	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5486	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5487	if (enmTrpmEvent == TRPM_TRAP)
5488	{
5489	switch (uVector)
5490	{
5491	case X86_XCPT_BP:
5492	case X86_XCPT_OF:
5493	{
5494	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5495	break;
5496	}
5497
5498	case X86_XCPT_PF:
5499	case X86_XCPT_DF:
5500	case X86_XCPT_TS:
5501	case X86_XCPT_NP:
5502	case X86_XCPT_SS:
5503	case X86_XCPT_GP:
5504	case X86_XCPT_AC:
5505	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5506	/* no break! */
5507	default:
5508	{
5509	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5510	break;
5511	}
5512	}
5513	}
5514	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5515	{
5516	if (uVector != X86_XCPT_NMI)
5517	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5518	else
5519	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5520	}
5521	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5522	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5523	else
5524	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5525
5526	rc = TRPMResetTrap(pVCpu);
5527	AssertRC(rc);
5528	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5529	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5530	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5531	}
5532
5533
5534	/**
5535	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5536	* VT-x to execute any instruction.
5537	*
5538	* @param pvCpu Pointer to the VMCPU.
5539	*/
5540	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5541	{
5542	if (pVCpu->hm.s.Event.fPending)
5543	{
5544	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5545	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5546	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5547	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5548
5549	/* If a trap was already pending, we did something wrong! */
5550	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5551
5552	TRPMEVENT enmTrapType;
5553	switch (uVectorType)
5554	{
5555	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5556	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5557	enmTrapType = TRPM_HARDWARE_INT;
5558	break;
5559	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5560	enmTrapType = TRPM_SOFTWARE_INT;
5561	break;
5562	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5563	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5564	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5565	enmTrapType = TRPM_TRAP;
5566	break;
5567	default:
5568	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5569	enmTrapType = TRPM_32BIT_HACK;
5570	break;
5571	}
5572
5573	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5574	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5575	AssertRC(rc);
5576
5577	if (fErrorCodeValid)
5578	TRPMSetErrorCode(pVCpu, uErrorCode);
5579	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5580	&& uVector == X86_XCPT_PF)
5581	{
5582	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5583	}
5584	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5585	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5586	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5587	{
5588	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5589	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5590	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5591	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5592	}
5593	pVCpu->hm.s.Event.fPending = false;
5594	}
5595	}
5596
5597
5598	/**
5599	* Does the necessary state syncing before doing a longjmp to ring-3.
5600	*
5601	* @param pVM Pointer to the VM.
5602	* @param pVCpu Pointer to the VMCPU.
5603	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5604	* out-of-sync. Make sure to update the required fields
5605	* before using them.
5606	* @param rcExit The reason for exiting to ring-3. Can be
5607	* VINF_VMM_UNKNOWN_RING3_CALL.
5608	*
5609	* @remarks No-long-jmp zone!!!
5610	*/
5611	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5612	{
5613	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5614	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5615
5616	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5617	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5618	AssertRC(rc);
5619
5620	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5621	if (CPUMIsGuestFPUStateActive(pVCpu))
5622	{
5623	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5624	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5625	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5626	}
5627
5628	/* Restore debug registers if necessary and resync on next R0 reentry. */
5629	if (CPUMIsGuestDebugStateActive(pVCpu))
5630	{
5631	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5632	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5633	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5634	}
5635	else if (CPUMIsHyperDebugStateActive(pVCpu))
5636	{
5637	CPUMR0LoadHostDebugState(pVM, pVCpu);
5638	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5639	}
5640
5641	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5642	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5643	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5644	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5645	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5646	}
5647
5648
5649	/**
5650	* An action requires us to go back to ring-3. This function does the necessary
5651	* steps before we can safely return to ring-3. This is not the same as longjmps
5652	* to ring-3, this is voluntary.
5653	*
5654	* @param pVM Pointer to the VM.
5655	* @param pVCpu Pointer to the VMCPU.
5656	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5657	* out-of-sync. Make sure to update the required fields
5658	* before using them.
5659	* @param rcExit The reason for exiting to ring-3. Can be
5660	* VINF_VMM_UNKNOWN_RING3_CALL.
5661	*/
5662	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5663	{
5664	Assert(pVM);
5665	Assert(pVCpu);
5666	Assert(pMixedCtx);
5667	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5668
5669	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5670	{
5671	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5672	return;
5673	}
5674	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5675	{
5676	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5677	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5678	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5679	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5680	return;
5681	}
5682
5683	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5684	VMMRZCallRing3Disable(pVCpu);
5685	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5686
5687	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5688	hmR0VmxUpdateTRPM(pVCpu);
5689
5690	/* Sync. the guest state. */
5691	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5692	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5693
5694	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5695	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5696	\| CPUM_CHANGED_LDTR
5697	\| CPUM_CHANGED_GDTR
5698	\| CPUM_CHANGED_IDTR
5699	\| CPUM_CHANGED_TR
5700	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5701
5702	/* On our way back from ring-3 the following needs to be done. */
5703	/** @todo This can change with preemption hooks. */
5704	if (rcExit == VINF_EM_RAW_INTERRUPT)
5705	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5706	else
5707	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5708
5709	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5710	VMMRZCallRing3Enable(pVCpu);
5711	}
5712
5713
5714	/**
5715	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5716	* longjump to ring-3 and possibly get preempted.
5717	*
5718	* @param pVCpu Pointer to the VMCPU.
5719	* @param enmOperation The operation causing the ring-3 longjump.
5720	* @param pvUser The user argument (pointer to the possibly
5721	* out-of-date guest-CPU context).
5722	*
5723	* @remarks Must never be called with @a enmOperation ==
5724	* VMMCALLRING3_VM_R0_ASSERTION.
5725	*/
5726	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5727	{
5728	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5729	Assert(pVCpu);
5730	Assert(pvUser);
5731	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5732	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5733
5734	VMMRZCallRing3Disable(pVCpu);
5735	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5736	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5737	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5738	VMMRZCallRing3Enable(pVCpu);
5739	}
5740
5741
5742	/**
5743	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5744	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5745	*
5746	* @returns VBox status code.
5747	* @param pVCpu Pointer to the VMCPU.
5748	*/
5749	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5750	{
5751	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5752	{
5753	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5754	{
5755	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5756	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5757	AssertRC(rc);
5758	}
5759	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5760	}
5761
5762
5763	/**
5764	* Injects any pending events into the guest if the guest is in a state to
5765	* receive them.
5766	*
5767	* @returns VBox status code (informational status codes included).
5768	* @param pVCpu Pointer to the VMCPU.
5769	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5770	* out-of-sync. Make sure to update the required fields
5771	* before using them.
5772	*/
5773	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5774	{
5775	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5776	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5777	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5778	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5779
5780	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5781	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5782	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5783	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5784
5785	int rc = VINF_SUCCESS;
5786	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5787	{
5788	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5789	bool fInject = true;
5790	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5791	{
5792	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5793	AssertRCReturn(rc, rc);
5794	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5795	if ( fBlockInt
5796	\|\| fBlockSti
5797	\|\| fBlockMovSS)
5798	{
5799	fInject = false;
5800	}
5801	}
5802	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5803	&& ( fBlockMovSS
5804	\|\| fBlockSti))
5805	{
5806	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5807	fInject = false;
5808	}
5809
5810	if (fInject)
5811	{
5812	Log(("Injecting pending event\n"));
5813	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5814	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5815	AssertRCReturn(rc, rc);
5816	pVCpu->hm.s.Event.fPending = false;
5817	}
5818	else
5819	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5820	} /** @todo SMI. SMIs take priority over NMIs. */
5821	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5822	{
5823	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5824	if ( !fBlockMovSS
5825	&& !fBlockSti)
5826	{
5827	Log(("Injecting NMI\n"));
5828	RTGCUINTPTR uIntrInfo;
5829	uIntrInfo = X86_XCPT_NMI \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5830	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5831	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5832	0 /* GCPtrFaultAddress */, &uIntrState);
5833	AssertRCReturn(rc, rc);
5834	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5835	}
5836	else
5837	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5838	}
5839	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5840	{
5841	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5842	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5843	AssertRCReturn(rc, rc);
5844	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5845	if ( !fBlockInt
5846	&& !fBlockSti
5847	&& !fBlockMovSS)
5848	{
5849	uint8_t u8Interrupt;
5850	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5851	if (RT_SUCCESS(rc))
5852	{
5853	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5854	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5855	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5856	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5857	0 /* GCPtrFaultAddress */, &uIntrState);
5858	}
5859	else
5860	{
5861	/** @todo Does this actually happen? If not turn it into an assertion. */
5862	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5863	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5864	rc = VINF_SUCCESS;
5865	}
5866	}
5867	else
5868	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5869	}
5870
5871	/*
5872	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
5873	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
5874	*/
5875	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5876	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5877	int rc2 = VINF_SUCCESS;
5878	if ( fBlockSti
5879	\|\| fBlockMovSS)
5880	{
5881	if (!DBGFIsStepping(pVCpu))
5882	{
5883	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
5884	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
5885	{
5886	/*
5887	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
5888	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
5889	*/
5890	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
5891	}
5892	}
5893	else
5894	{
5895	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
5896	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
5897	uIntrState = 0;
5898	}
5899	}
5900
5901	/*
5902	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5903	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5904	*/
5905	rc2 \|= hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
5906	AssertRC(rc2);
5907
5908	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
5909	return rc;
5910	}
5911
5912
5913	/**
5914	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
5915	*
5916	* @param pVCpu Pointer to the VMCPU.
5917	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5918	* out-of-sync. Make sure to update the required fields
5919	* before using them.
5920	*/
5921	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5922	{
5923	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5924	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5925	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5926	}
5927
5928
5929	/**
5930	* Injects a double-fault (#DF) exception into the VM.
5931	*
5932	* @returns VBox status code (informational status code included).
5933	* @param pVCpu Pointer to the VMCPU.
5934	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5935	* out-of-sync. Make sure to update the required fields
5936	* before using them.
5937	*/
5938	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
5939	{
5940	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5941	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5942	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5943	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5944	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
5945	puIntrState);
5946	}
5947
5948
5949	/**
5950	* Sets a debug (#DB) exception as pending-for-injection into the VM.
5951	*
5952	* @param pVCpu Pointer to the VMCPU.
5953	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5954	* out-of-sync. Make sure to update the required fields
5955	* before using them.
5956	*/
5957	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5958	{
5959	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5960	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5961	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5962	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
5963	}
5964
5965
5966	/**
5967	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
5968	*
5969	* @param pVCpu Pointer to the VMCPU.
5970	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5971	* out-of-sync. Make sure to update the required fields
5972	* before using them.
5973	* @param cbInstr The value of RIP that is to be pushed on the guest
5974	* stack.
5975	*/
5976	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5977	{
5978	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5979	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5980	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5981	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
5982	}
5983
5984
5985	/**
5986	* Injects a general-protection (#GP) fault into the VM.
5987	*
5988	* @returns VBox status code (informational status code included).
5989	* @param pVCpu Pointer to the VMCPU.
5990	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5991	* out-of-sync. Make sure to update the required fields
5992	* before using them.
5993	* @param u32ErrorCode The error code associated with the #GP.
5994	*/
5995	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
5996	uint32_t *puIntrState)
5997	{
5998	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5999	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6000	if (fErrorCodeValid)
6001	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6002	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6003	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6004	puIntrState);
6005	}
6006
6007
6008	/**
6009	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6010	*
6011	* @param pVCpu Pointer to the VMCPU.
6012	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6013	* out-of-sync. Make sure to update the required fields
6014	* before using them.
6015	* @param uVector The software interrupt vector number.
6016	* @param cbInstr The value of RIP that is to be pushed on the guest
6017	* stack.
6018	*/
6019	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6020	{
6021	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6022	if ( uVector == X86_XCPT_BP
6023	\|\| uVector == X86_XCPT_OF)
6024	{
6025	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6026	}
6027	else
6028	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6029	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6030	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6031	}
6032
6033
6034	/**
6035	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6036	* stack.
6037	*
6038	* @returns VBox status code (information status code included).
6039	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6040	* @param pVM Pointer to the VM.
6041	* @param pMixedCtx Pointer to the guest-CPU context.
6042	* @param uValue The value to push to the guest stack.
6043	*/
6044	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6045	{
6046	/*
6047	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6048	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6049	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6050	*/
6051	if (pMixedCtx->sp == 1)
6052	return VINF_EM_RESET;
6053	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6054	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6055	AssertRCReturn(rc, rc);
6056	return rc;
6057	}
6058
6059
6060	/**
6061	* Injects an event into the guest upon VM-entry by updating the relevant fields
6062	* in the VM-entry area in the VMCS.
6063	*
6064	* @returns VBox status code (informational error codes included).
6065	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6066	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6067	*
6068	* @param pVCpu Pointer to the VMCPU.
6069	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6070	* be out-of-sync. Make sure to update the required
6071	* fields before using them.
6072	* @param u64IntrInfo The VM-entry interruption-information field.
6073	* @param cbInstr The VM-entry instruction length in bytes (for
6074	* software interrupts, exceptions and privileged
6075	* software exceptions).
6076	* @param u32ErrCode The VM-entry exception error code.
6077	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6078	* @param puIntrState Pointer to the current guest interruptibility-state.
6079	* This interruptibility-state will be updated if
6080	* necessary. This cannot not be NULL.
6081	*
6082	* @remarks No-long-jump zone!!!
6083	* @remarks Requires CR0!
6084	*/
6085	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6086	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6087	{
6088	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6089	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6090	Assert(puIntrState);
6091	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6092
6093	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6094	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6095
6096	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6097	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6098	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6099
6100	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6101
6102	/* We require CR0 to check if the guest is in real-mode. */
6103	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6104	AssertRCReturn(rc, rc);
6105
6106	/*
6107	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6108	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6109	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6110	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6111	*/
6112	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6113	{
6114	PVM pVM = pVCpu->CTX_SUFF(pVM);
6115	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6116	{
6117	Assert(PDMVmmDevHeapIsEnabled(pVM));
6118	Assert(pVM->hm.s.vmx.pRealModeTSS);
6119
6120	/* Save the required guest state bits from the VMCS. */
6121	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6122	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6123	AssertRCReturn(rc, rc);
6124
6125	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6126	const size_t cbIdtEntry = 4;
6127	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6128	{
6129	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6130	if (uVector == X86_XCPT_DF)
6131	return VINF_EM_RESET;
6132	else if (uVector == X86_XCPT_GP)
6133	{
6134	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6135	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6136	}
6137
6138	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6139	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6140	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6141	}
6142
6143	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6144	uint16_t uGuestIp = pMixedCtx->ip;
6145	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6146	{
6147	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6148	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6149	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6150	}
6151	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6152	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6153
6154	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6155	uint16_t offIdtEntry = 0;
6156	RTSEL selIdtEntry = 0;
6157	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6158	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6159	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6160	AssertRCReturn(rc, rc);
6161
6162	/* Construct the stack frame for the interrupt/exception handler. */
6163	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6164	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6165	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6166	AssertRCReturn(rc, rc);
6167
6168	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6169	if (rc == VINF_SUCCESS)
6170	{
6171	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6172	pMixedCtx->rip = offIdtEntry;
6173	pMixedCtx->cs.Sel = selIdtEntry;
6174	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6175	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6176	&& uVector == X86_XCPT_PF)
6177	{
6178	pMixedCtx->cr2 = GCPtrFaultAddress;
6179	}
6180	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6181	\| HM_CHANGED_GUEST_RIP
6182	\| HM_CHANGED_GUEST_RFLAGS
6183	\| HM_CHANGED_GUEST_RSP;
6184
6185	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6186	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6187	{
6188	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6189	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6190	Log(("Clearing inhibition due to STI.\n"));
6191	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6192	}
6193	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6194	}
6195	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6196	return rc;
6197	}
6198	else
6199	{
6200	/*
6201	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6202	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6203	*/
6204	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6205	}
6206	}
6207
6208	/* Validate. */
6209	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6210	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6211	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6212
6213	/* Inject. */
6214	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6215	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6216	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6217	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6218
6219	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6220	&& uVector == X86_XCPT_PF)
6221	{
6222	pMixedCtx->cr2 = GCPtrFaultAddress;
6223	}
6224	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RGv\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6225
6226	AssertRCReturn(rc, rc);
6227	return rc;
6228	}
6229
6230
6231	/**
6232	* Enters the VT-x session.
6233	*
6234	* @returns VBox status code.
6235	* @param pVM Pointer to the VM.
6236	* @param pVCpu Pointer to the VMCPU.
6237	* @param pCpu Pointer to the CPU info struct.
6238	*/
6239	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6240	{
6241	AssertPtr(pVM);
6242	AssertPtr(pVCpu);
6243	Assert(pVM->hm.s.vmx.fSupported);
6244	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6245	NOREF(pCpu);
6246
6247	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6248
6249	/* Make sure we're in VMX root mode. */
6250	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6251	if (!(u32HostCR4 & X86_CR4_VMXE))
6252	{
6253	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6254	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6255	}
6256
6257	/* Load the active VMCS as the current one. */
6258	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6259	if (RT_FAILURE(rc))
6260	return rc;
6261
6262	/** @todo this will change with preemption hooks where can can VMRESUME as long
6263	* as we're no preempted. */
6264	pVCpu->hm.s.fResumeVM = false;
6265	return VINF_SUCCESS;
6266	}
6267
6268
6269	/**
6270	* Leaves the VT-x session.
6271	*
6272	* @returns VBox status code.
6273	* @param pVM Pointer to the VM.
6274	* @param pVCpu Pointer to the VMCPU.
6275	* @param pCtx Pointer to the guest-CPU context.
6276	*/
6277	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6278	{
6279	AssertPtr(pVCpu);
6280	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6281	NOREF(pVM);
6282	NOREF(pCtx);
6283
6284	/** @todo this will change with preemption hooks where we only VMCLEAR when
6285	* we are actually going to be preempted, not all the time like we
6286	* currently do. */
6287	/*
6288	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6289	* and mark the VMCS launch-state as "clear".
6290	*/
6291	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6292	return rc;
6293	}
6294
6295
6296	/**
6297	* Saves the host state in the VMCS host-state.
6298	* Sets up the VM-exit MSR-load area.
6299	*
6300	* The CPU state will be loaded from these fields on every successful VM-exit.
6301	*
6302	* @returns VBox status code.
6303	* @param pVM Pointer to the VM.
6304	* @param pVCpu Pointer to the VMCPU.
6305	*
6306	* @remarks No-long-jump zone!!!
6307	*/
6308	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6309	{
6310	AssertPtr(pVM);
6311	AssertPtr(pVCpu);
6312	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6313
6314	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6315
6316	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6317	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6318	return VINF_SUCCESS;
6319
6320	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6321	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6322
6323	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6324	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6325
6326	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6327	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6328
6329	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6330	return rc;
6331	}
6332
6333
6334	/**
6335	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6336	* loaded from these fields on every successful VM-entry.
6337	*
6338	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6339	* Sets up the VM-entry controls.
6340	* Sets up the appropriate VMX non-root function to execute guest code based on
6341	* the guest CPU mode.
6342	*
6343	* @returns VBox status code.
6344	* @param pVM Pointer to the VM.
6345	* @param pVCpu Pointer to the VMCPU.
6346	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6347	* out-of-sync. Make sure to update the required fields
6348	* before using them.
6349	*
6350	* @remarks No-long-jump zone!!!
6351	*/
6352	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6353	{
6354	AssertPtr(pVM);
6355	AssertPtr(pVCpu);
6356	AssertPtr(pMixedCtx);
6357	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6358
6359	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6360
6361	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6362
6363	/* Determine real-on-v86 mode. */
6364	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6365	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6366	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6367	{
6368	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6369	}
6370
6371	/** @todo if the order of loading is important, inform it via comments here */
6372	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6373	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6374
6375	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6376	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6377
6378	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6379	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6380
6381	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6382	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6383
6384	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6385	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6386
6387	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6388	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6389
6390	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6391	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6392
6393	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6394	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6395
6396	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6397	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6398
6399	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6400	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6401
6402	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6403	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6404	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6405
6406	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6407	return rc;
6408	}
6409
6410
6411	/**
6412	* Does the preparations before executing guest code in VT-x.
6413	*
6414	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6415	* recompiler. We must be cautious what we do here regarding committing
6416	* guest-state information into the the VMCS assuming we assuredly execute the
6417	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6418	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6419	* that the recompiler can (and should) use them when it resumes guest
6420	* execution. Otherwise such operations must be done when we can no longer
6421	* exit to ring-3.
6422	*
6423	* @returns VBox status code (informational status codes included).
6424	* @retval VINF_SUCCESS if we can proceed with running the guest.
6425	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6426	* into the guest.
6427	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6428	*
6429	* @param pVM Pointer to the VM.
6430	* @param pVCpu Pointer to the VMCPU.
6431	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6432	* out-of-sync. Make sure to update the required fields
6433	* before using them.
6434	* @param pVmxTransient Pointer to the VMX transient structure.
6435	*
6436	* @remarks Called with preemption disabled.
6437	*/
6438	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6439	{
6440	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6441
6442	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6443	PGMRZDynMapFlushAutoSet(pVCpu);
6444	#endif
6445
6446	/* Check force flag actions that might require us to go back to ring-3. */
6447	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6448	if (rc != VINF_SUCCESS)
6449	return rc;
6450
6451	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6452	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6453	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6454	{
6455	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6456	RTGCPHYS GCPhysApicBase;
6457	GCPhysApicBase = pMixedCtx->msrApicBase;
6458	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6459
6460	/* Unalias any existing mapping. */
6461	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6462	AssertRCReturn(rc, rc);
6463
6464	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6465	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6466	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6467	AssertRCReturn(rc, rc);
6468
6469	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6470	}
6471
6472	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6473	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6474	pVmxTransient->uEFlags = ASMIntDisableFlags();
6475	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6476	{
6477	ASMSetFlags(pVmxTransient->uEFlags);
6478	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6479	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6480	return VINF_EM_RAW_INTERRUPT;
6481	}
6482	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6483	#endif
6484
6485	/*
6486	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6487	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6488	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6489	*/
6490	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6491	AssertRCReturn(rc, rc);
6492	return rc;
6493	}
6494
6495
6496	/**
6497	* Prepares to run guest code in VT-x and we've committed to doing so. This
6498	* means there is no backing out to ring-3 or anywhere else at this
6499	* point.
6500	*
6501	* @param pVM Pointer to the VM.
6502	* @param pVCpu Pointer to the VMCPU.
6503	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6504	* out-of-sync. Make sure to update the required fields
6505	* before using them.
6506	* @param pVmxTransient Pointer to the VMX transient structure.
6507	*
6508	* @remarks Called with preemption disabled.
6509	* @remarks No-long-jump zone!!!
6510	*/
6511	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6512	{
6513	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6514	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6515
6516	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6517	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6518	pVmxTransient->uEFlags = ASMIntDisableFlags();
6519	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6520	#endif
6521
6522	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6523	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6524	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6525	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6526	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6527	#endif
6528	int rc = VINF_SUCCESS;
6529	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6530	{
6531	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6532	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6533	}
6534	else if (pVCpu->hm.s.fContextUseFlags)
6535	{
6536	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6537	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6538	}
6539	AssertRC(rc);
6540	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6541
6542	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6543	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6544	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6545
6546	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6547	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6548
6549	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6550	{
6551	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6552	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6553	}
6554
6555	/*
6556	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6557	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6558	*/
6559	if (pVM->hm.s.fTPRPatchingActive)
6560	{
6561	Assert(!CPUMIsGuestInLongMode(pVCpu));
6562
6563	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6564	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6565	AssertRC(rc);
6566
6567	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6568	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6569	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6570	see hmR0VmxLoadGuestApicState(). */
6571	}
6572
6573	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6574	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6575	to start executing. */
6576	}
6577
6578
6579	/**
6580	* Performs some essential restoration of state after running guest code in
6581	* VT-x.
6582	*
6583	* @param pVM Pointer to the VM.
6584	* @param pVCpu Pointer to the VMCPU.
6585	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6586	* out-of-sync. Make sure to update the required fields
6587	* before using them.
6588	* @param pVmxTransient Pointer to the VMX transient structure.
6589	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6590	*
6591	* @remarks Called with interrupts disabled.
6592	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6593	* unconditionally when it is safe to do so.
6594	*/
6595	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6596	{
6597	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6598	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6599
6600	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6601	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6602	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6603	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6604	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6605
6606	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6607	{
6608	/** @todo Find a way to fix hardcoding a guestimate. */
6609	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6610	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6611	}
6612
6613	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6614	Assert(!(ASMGetFlags() & X86_EFL_IF));
6615	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6616
6617	/* Restore the effects of TPR patching if any. */
6618	if (pVM->hm.s.fTPRPatchingActive)
6619	{
6620	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6621	AssertRC(rc);
6622	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6623	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6624	}
6625
6626	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6627	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6628
6629	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6630	uint32_t uExitReason;
6631	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6632	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6633	AssertRC(rc);
6634	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6635	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6636
6637	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6638	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6639
6640	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6641	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6642	{
6643	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6644	return;
6645	}
6646
6647	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6648	{
6649	/* Update the guest interruptibility-state from the VMCS. */
6650	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6651	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6652	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6653	AssertRC(rc);
6654	#endif
6655	/*
6656	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6657	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6658	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6659	*/
6660	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6661	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6662	{
6663	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6664	AssertRC(rc);
6665	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6666	}
6667	}
6668	}
6669
6670
6671	/**
6672	* Runs the guest code using VT-x.
6673	*
6674	* @returns VBox status code.
6675	* @param pVM Pointer to the VM.
6676	* @param pVCpu Pointer to the VMCPU.
6677	* @param pCtx Pointer to the guest-CPU context.
6678	*
6679	* @remarks Called with preemption disabled.
6680	*/
6681	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6682	{
6683	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6684	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6685
6686	VMXTRANSIENT VmxTransient;
6687	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6688	int rc = VERR_INTERNAL_ERROR_5;
6689	uint32_t cLoops = 0;
6690	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6691
6692	for (;; cLoops++)
6693	{
6694	Assert(!HMR0SuspendPending());
6695	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6696	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6697	(unsigned)RTMpCpuId(), cLoops));
6698
6699	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6700	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6701	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6702	if (rc != VINF_SUCCESS)
6703	break;
6704
6705	/*
6706	* No longjmps to ring-3 from this point on!!!
6707	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6708	* This also disables flushing of the R0-logger instance (if any).
6709	*/
6710	VMMRZCallRing3Disable(pVCpu);
6711	VMMRZCallRing3RemoveNotification(pVCpu);
6712	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6713
6714	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6715	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6716
6717	/*
6718	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6719	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6720	*/
6721	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6722	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6723	{
6724	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6725	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6726	return rc;
6727	}
6728
6729	/* Handle the VM-exit. */
6730	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6731	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6732	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6733	#ifdef HM_PROFILE_EXIT_DISPATCH
6734	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed);
6735	#endif
6736	#ifdef HMVMX_USE_FUNCTION_TABLE
6737	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6738	#else
6739	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6740	#endif
6741	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6742	if (rc != VINF_SUCCESS)
6743	break;
6744	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6745	{
6746	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6747	rc = VINF_EM_RAW_INTERRUPT;
6748	break;
6749	}
6750	}
6751
6752	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6753	if (rc == VERR_EM_INTERPRETER)
6754	rc = VINF_EM_RAW_EMULATE_INSTR;
6755	else if (rc == VINF_EM_RESET)
6756	rc = VINF_EM_TRIPLE_FAULT;
6757	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6758	return rc;
6759	}
6760
6761
6762	#ifndef HMVMX_USE_FUNCTION_TABLE
6763	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6764	{
6765	int rc;
6766	switch (rcReason)
6767	{
6768	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6769	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6770	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6771	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6772	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6773	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6774	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6775	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6776	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6777	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6778	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6779	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6780	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6781	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6782	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6783	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6784	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6785	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6786	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6787	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6788	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6789	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6790	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6791	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6792	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6793	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6794	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6795	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6796	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6797	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6798	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6799	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6800	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6801
6802	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6803	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6804	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6805	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6806	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6807	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6808	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6809	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6810	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6811
6812	case VMX_EXIT_VMCALL:
6813	case VMX_EXIT_VMCLEAR:
6814	case VMX_EXIT_VMLAUNCH:
6815	case VMX_EXIT_VMPTRLD:
6816	case VMX_EXIT_VMPTRST:
6817	case VMX_EXIT_VMREAD:
6818	case VMX_EXIT_VMRESUME:
6819	case VMX_EXIT_VMWRITE:
6820	case VMX_EXIT_VMXOFF:
6821	case VMX_EXIT_VMXON:
6822	case VMX_EXIT_INVEPT:
6823	case VMX_EXIT_INVVPID:
6824	case VMX_EXIT_VMFUNC:
6825	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6826	break;
6827	default:
6828	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6829	break;
6830	}
6831	return rc;
6832	}
6833	#endif
6834
6835
6836	/** Profiling macro. */
6837	#ifdef HM_PROFILE_EXIT_DISPATCH
6838	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
6839	#else
6840	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
6841	#endif
6842
6843
6844	#ifdef DEBUG
6845	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6846	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6847	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6848	# define VMX_ASSERT_PREEMPT_CPUID() \
6849	do \
6850	{ \
6851	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6852	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6853	} while (0)
6854
6855	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6856	do { \
6857	AssertPtr(pVCpu); \
6858	AssertPtr(pMixedCtx); \
6859	AssertPtr(pVmxTransient); \
6860	Assert(pVmxTransient->fVMEntryFailed == false); \
6861	Assert(ASMIntAreEnabled()); \
6862	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6863	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6864	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
6865	(unsigned)pVCpu->idCpu)); \
6866	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6867	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6868	VMX_ASSERT_PREEMPT_CPUID(); \
6869	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
6870	} while (0)
6871	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6872	do { \
6873	LogFunc(("\n")); \
6874	} while(0)
6875	#else /* Release builds */
6876	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
6877	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6878	#endif
6879
6880
6881	/**
6882	* Advances the guest RIP after reading it from the VMCS.
6883	*
6884	* @returns VBox status code.
6885	* @param pVCpu Pointer to the VMCPU.
6886	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6887	* out-of-sync. Make sure to update the required fields
6888	* before using them.
6889	* @param pVmxTransient Pointer to the VMX transient structure.
6890	*
6891	* @remarks No-long-jump zone!!!
6892	*/
6893	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6894	{
6895	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6896	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6897	AssertRCReturn(rc, rc);
6898
6899	pMixedCtx->rip += pVmxTransient->cbInstr;
6900	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6901	return rc;
6902	}
6903
6904
6905	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6906	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6907	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6908	/**
6909	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6910	*/
6911	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6912	{
6913	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6915	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6916	Assert(ASMIntAreEnabled());
6917	return VINF_SUCCESS;
6918	#else
6919	return VINF_EM_RAW_INTERRUPT;
6920	#endif
6921	}
6922
6923
6924	/**
6925	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6926	*/
6927	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6928	{
6929	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6930	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6931
6932	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6933	AssertRCReturn(rc, rc);
6934
6935	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6936	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6937	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6938
6939	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6940	{
6941	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6942	return VINF_EM_RAW_INTERRUPT;
6943	}
6944
6945	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6946	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6947	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6948	{
6949	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6950	return VINF_SUCCESS;
6951	}
6952	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6953	{
6954	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6955	return rc;
6956	}
6957
6958	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6959	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6960	switch (uIntrType)
6961	{
6962	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6963	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6964	/* no break */
6965	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
6966	{
6967	switch (uVector)
6968	{
6969	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
6970	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
6971	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
6972	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
6973	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
6974	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
6975	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
6976	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6977	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6978	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6979	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6980	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6981	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6982	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6983	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6984	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6985	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
6986	#endif
6987	default:
6988	{
6989	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6990	AssertRCReturn(rc, rc);
6991
6992	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6993	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6994	{
6995	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6996	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
6997	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6998	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
6999	AssertRCReturn(rc, rc);
7000	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7001	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7002	0 /* GCPtrFaultAddress */);
7003	AssertRCReturn(rc, rc);
7004	}
7005	else
7006	{
7007	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7008	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7009	}
7010	break;
7011	}
7012	}
7013	break;
7014	}
7015
7016	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7017	default:
7018	{
7019	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7020	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7021	break;
7022	}
7023	}
7024	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7025	return rc;
7026	}
7027
7028
7029	/**
7030	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7031	*/
7032	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7033	{
7034	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7035
7036	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7037	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7038	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7039	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7040	AssertRCReturn(rc, rc);
7041
7042	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7043	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7044	return VINF_SUCCESS;
7045	}
7046
7047
7048	/**
7049	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7050	*/
7051	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7052	{
7053	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7054	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7055	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7056	}
7057
7058
7059	/**
7060	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7061	*/
7062	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7063	{
7064	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7066	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7067	}
7068
7069
7070	/**
7071	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7072	*/
7073	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7074	{
7075	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7076	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7077	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7078	}
7079
7080
7081	/**
7082	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7083	*/
7084	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7085	{
7086	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7087	PVM pVM = pVCpu->CTX_SUFF(pVM);
7088	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7089	if (RT_LIKELY(rc == VINF_SUCCESS))
7090	{
7091	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7092	Assert(pVmxTransient->cbInstr == 2);
7093	}
7094	else
7095	{
7096	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7097	rc = VERR_EM_INTERPRETER;
7098	}
7099	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7100	return rc;
7101	}
7102
7103
7104	/**
7105	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7106	*/
7107	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7108	{
7109	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7110	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7111	AssertRCReturn(rc, rc);
7112
7113	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7114	return VINF_EM_RAW_EMULATE_INSTR;
7115
7116	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7117	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7118	}
7119
7120
7121	/**
7122	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7123	*/
7124	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7125	{
7126	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7127	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7128	AssertRCReturn(rc, rc);
7129
7130	PVM pVM = pVCpu->CTX_SUFF(pVM);
7131	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7132	if (RT_LIKELY(rc == VINF_SUCCESS))
7133	{
7134	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7135	Assert(pVmxTransient->cbInstr == 2);
7136	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7137	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7138	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7139	}
7140	else
7141	{
7142	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7143	rc = VERR_EM_INTERPRETER;
7144	}
7145	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7146	return rc;
7147	}
7148
7149
7150	/**
7151	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7152	*/
7153	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7154	{
7155	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7156	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7157	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7158	AssertRCReturn(rc, rc);
7159
7160	PVM pVM = pVCpu->CTX_SUFF(pVM);
7161	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7162	if (RT_LIKELY(rc == VINF_SUCCESS))
7163	{
7164	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7165	Assert(pVmxTransient->cbInstr == 3);
7166	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7167	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7168	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7169	}
7170	else
7171	{
7172	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7173	rc = VERR_EM_INTERPRETER;
7174	}
7175	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7176	return rc;
7177	}
7178
7179
7180	/**
7181	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7182	*/
7183	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7184	{
7185	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7186	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7187	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7188	AssertRCReturn(rc, rc);
7189
7190	PVM pVM = pVCpu->CTX_SUFF(pVM);
7191	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7192	if (RT_LIKELY(rc == VINF_SUCCESS))
7193	{
7194	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7195	Assert(pVmxTransient->cbInstr == 2);
7196	}
7197	else
7198	{
7199	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7200	rc = VERR_EM_INTERPRETER;
7201	}
7202	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7203	return rc;
7204	}
7205
7206
7207	/**
7208	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7209	*/
7210	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7211	{
7212	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7213	PVM pVM = pVCpu->CTX_SUFF(pVM);
7214	Assert(!pVM->hm.s.fNestedPaging);
7215
7216	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7217	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7218	AssertRCReturn(rc, rc);
7219
7220	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7221	rc = VBOXSTRICTRC_VAL(rc2);
7222	if (RT_LIKELY(rc == VINF_SUCCESS))
7223	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7224	else
7225	{
7226	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RGv failed with %Rrc\n",
7227	pVmxTransient->uExitQualification, rc));
7228	}
7229	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7230	return rc;
7231	}
7232
7233
7234	/**
7235	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7236	*/
7237	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7238	{
7239	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7240	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7241	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7242	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7243	AssertRCReturn(rc, rc);
7244
7245	PVM pVM = pVCpu->CTX_SUFF(pVM);
7246	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7247	if (RT_LIKELY(rc == VINF_SUCCESS))
7248	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7249	else
7250	{
7251	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7252	rc = VERR_EM_INTERPRETER;
7253	}
7254	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7255	return rc;
7256	}
7257
7258
7259	/**
7260	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7261	*/
7262	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7263	{
7264	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7265	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7266	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7267	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7268	AssertRCReturn(rc, rc);
7269
7270	PVM pVM = pVCpu->CTX_SUFF(pVM);
7271	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7272	rc = VBOXSTRICTRC_VAL(rc2);
7273	if (RT_LIKELY( rc == VINF_SUCCESS
7274	\|\| rc == VINF_EM_HALT))
7275	{
7276	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7277	AssertRCReturn(rc3, rc3);
7278
7279	if ( rc == VINF_EM_HALT
7280	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7281	{
7282	rc = VINF_SUCCESS;
7283	}
7284	}
7285	else
7286	{
7287	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7288	rc = VERR_EM_INTERPRETER;
7289	}
7290	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7291	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7292	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7293	return rc;
7294	}
7295
7296
7297	/**
7298	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7299	*/
7300	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7301	{
7302	/*
7303	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7304	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7305	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7306	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7307	*/
7308	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7309	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7310	}
7311
7312
7313	/**
7314	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7315	*/
7316	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7317	{
7318	/*
7319	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7320	* root operation. If we get there there is something funny going on.
7321	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7322	*/
7323	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7324	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7325	}
7326
7327
7328	/**
7329	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7330	*/
7331	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7332	{
7333	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7334	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7335	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7336	}
7337
7338
7339	/**
7340	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7341	*/
7342	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7343	{
7344	/*
7345	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7346	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7347	* See Intel spec. 25.3 "Other Causes of VM-exits".
7348	*/
7349	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7350	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7351	}
7352
7353
7354	/**
7355	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7356	* VM-exit.
7357	*/
7358	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7359	{
7360	/*
7361	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7362	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7363	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7364	*/
7365	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7366	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7367	}
7368
7369
7370	/**
7371	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7372	* VM-exit.
7373	*/
7374	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7375	{
7376	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7377	return VINF_EM_RESET;
7378	}
7379
7380
7381	/**
7382	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7383	*/
7384	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7385	{
7386	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7387	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7388	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7389	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7390	AssertRCReturn(rc, rc);
7391
7392	pMixedCtx->rip++;
7393	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7394	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7395	rc = VINF_SUCCESS;
7396	else
7397	rc = VINF_EM_HALT;
7398
7399	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7400	return rc;
7401	}
7402
7403
7404	/**
7405	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7406	*/
7407	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7408	{
7409	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7410	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7411	return VINF_SUCCESS;
7412	}
7413
7414
7415	/**
7416	* VM-exit handler for expiry of the VMX preemption timer.
7417	*/
7418	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7419	{
7420	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7421
7422	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7423	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7424	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7425
7426	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7427	PVM pVM = pVCpu->CTX_SUFF(pVM);
7428	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7429	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7430	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7431	}
7432
7433
7434	/**
7435	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7436	*/
7437	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7438	{
7439	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7440	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7441	/** @todo check if XSETBV is supported by the recompiler. */
7442	return VERR_EM_INTERPRETER;
7443	}
7444
7445
7446	/**
7447	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7448	*/
7449	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7450	{
7451	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7452	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7453	/** @todo implement EMInterpretInvpcid() */
7454	return VERR_EM_INTERPRETER;
7455	}
7456
7457
7458	/**
7459	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7460	* Error VM-exit.
7461	*/
7462	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	uint32_t uIntrState;
7465	RTHCUINTREG uHCReg;
7466	uint64_t u64Val;
7467	uint32_t u32Val;
7468
7469	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7470	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7471	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7472	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7473	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7474	AssertRCReturn(rc, rc);
7475
7476	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7477	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7478	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7479	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7480
7481	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7482	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7483	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7484	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7485	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7486	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7487	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7488	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7489	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7490	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7491	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7492	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7493
7494	PVM pVM = pVCpu->CTX_SUFF(pVM);
7495	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7496
7497	return VERR_VMX_INVALID_GUEST_STATE;
7498	}
7499
7500
7501	/**
7502	* VM-exit handler for VM-entry failure due to an MSR-load
7503	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7504	*/
7505	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7506	{
7507	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7508	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7509	}
7510
7511
7512	/**
7513	* VM-exit handler for VM-entry failure due to a machine-check event
7514	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7515	*/
7516	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7517	{
7518	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7519	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7520	}
7521
7522
7523	/**
7524	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7525	* theory.
7526	*/
7527	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7528	{
7529	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7530	return VERR_VMX_UNDEFINED_EXIT_CODE;
7531	}
7532
7533
7534	/**
7535	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7536	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7537	* Conditional VM-exit.
7538	*/
7539	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7540	{
7541	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7542	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7543	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7544	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7545	return VERR_EM_INTERPRETER;
7546	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7547	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7548	}
7549
7550
7551	/**
7552	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7553	*/
7554	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7555	{
7556	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7557	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7558	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7559	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7560	return VERR_EM_INTERPRETER;
7561	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7562	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7563	}
7564
7565
7566	/**
7567	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7568	*/
7569	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7570	{
7571	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7572	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7573	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7574	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7575	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7576	AssertRCReturn(rc, rc);
7577
7578	PVM pVM = pVCpu->CTX_SUFF(pVM);
7579	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7580	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7581	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7582	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7583
7584	if (RT_LIKELY(rc == VINF_SUCCESS))
7585	{
7586	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7587	Assert(pVmxTransient->cbInstr == 2);
7588	}
7589	return rc;
7590	}
7591
7592
7593	/**
7594	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7595	*/
7596	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7597	{
7598	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7599	PVM pVM = pVCpu->CTX_SUFF(pVM);
7600	int rc = VINF_SUCCESS;
7601
7602	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7603	if ( pVM->hm.s.fTPRPatchingActive
7604	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7605	{
7606	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7607	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7608	{
7609	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7610	AssertRC(rc);
7611	}
7612
7613	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7614	Assert(pVmxTransient->cbInstr == 2);
7615	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7616	return VINF_SUCCESS;
7617	}
7618
7619	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7620	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7621	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7622	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7623	AssertRCReturn(rc, rc);
7624	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7625
7626	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7627	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7628	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7629
7630	if (RT_LIKELY(rc == VINF_SUCCESS))
7631	{
7632	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7633
7634	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7635	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7636	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7637	{
7638	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7639	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7640	}
7641	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7642	{
7643	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7644	AssertRCReturn(rc, rc);
7645	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7646	}
7647
7648	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7649	if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)))
7650	{
7651	switch (pMixedCtx->ecx)
7652	{
7653	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7654	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7655	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7656	case MSR_K8_KERNEL_GS_BASE: /* If we auto-load it, update HM_CHANGED_VMX_GUEST_AUTO_MSRS. */ break;
7657	case MSR_K8_FS_BASE: /* no break */
7658	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7659	}
7660	}
7661	#ifdef VBOX_STRICT
7662	else
7663	{
7664	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7665	switch (pMixedCtx->ecx)
7666	{
7667	case MSR_IA32_SYSENTER_CS:
7668	case MSR_IA32_SYSENTER_EIP:
7669	case MSR_IA32_SYSENTER_ESP:
7670	case MSR_K8_FS_BASE:
7671	case MSR_K8_GS_BASE:
7672	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7673	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7674
7675	case MSR_K8_LSTAR:
7676	case MSR_K6_STAR:
7677	case MSR_K8_SF_MASK:
7678	case MSR_K8_TSC_AUX:
7679	{
7680	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7681	pMixedCtx->ecx));
7682	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7683	}
7684
7685	case MSR_K8_KERNEL_GS_BASE:
7686	{
7687	AssertMsgFailed(("Unexpected WRMSR for an MSR that is manually loaded/stored on every VM-exit. ecx=%#RX32\n",
7688	pMixedCtx->ecx));
7689	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7690	}
7691	}
7692	}
7693	#endif /* VBOX_STRICT */
7694	}
7695	return rc;
7696	}
7697
7698
7699	/**
7700	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7701	*/
7702	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7703	{
7704	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7705	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7707	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7708	return VERR_EM_INTERPRETER;
7709	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7710	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7711	}
7712
7713
7714	/**
7715	* VM-exit handler for when the TPR value is lowered below the specified
7716	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7717	*/
7718	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7719	{
7720	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7721	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7722
7723	/*
7724	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7725	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7726	* resume guest execution.
7727	*/
7728	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7729	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7730	return VINF_SUCCESS;
7731	}
7732
7733
7734	/**
7735	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7736	* VM-exit.
7737	*
7738	* @retval VINF_SUCCESS when guest execution can continue.
7739	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7740	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7741	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7742	* recompiler.
7743	*/
7744	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7745	{
7746	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7747	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7748	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7749	AssertRCReturn(rc, rc);
7750
7751	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7752	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7753	PVM pVM = pVCpu->CTX_SUFF(pVM);
7754	switch (uAccessType)
7755	{
7756	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7757	{
7758	#if 0
7759	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7760	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7761	#else
7762	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7763	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7764	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7765	#endif
7766	AssertRCReturn(rc, rc);
7767
7768	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7769	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7770	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7771	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7772
7773	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7774	{
7775	case 0: /* CR0 */
7776	Log(("CR0 write rc=%d CR0=%#RGv\n", rc, pMixedCtx->cr0));
7777	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7778	break;
7779	case 2: /* C2 **/
7780	/* Nothing to do here, CR2 it's not part of the VMCS. */
7781	break;
7782	case 3: /* CR3 */
7783	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7784	Log(("CR3 write rc=%d CR3=%#RGv\n", rc, pMixedCtx->cr3));
7785	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7786	break;
7787	case 4: /* CR4 */
7788	Log(("CR4 write rc=%d CR4=%#RGv\n", rc, pMixedCtx->cr4));
7789	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7790	break;
7791	case 8: /* CR8 */
7792	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7793	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7794	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7795	break;
7796	default:
7797	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7798	break;
7799	}
7800
7801	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7802	break;
7803	}
7804
7805	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7806	{
7807	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7808	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7809	AssertRCReturn(rc, rc);
7810	Assert( !pVM->hm.s.fNestedPaging
7811	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7812	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7813
7814	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7815	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7816	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7817
7818	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7819	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7820	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7821	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7822	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7823	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7824	break;
7825	}
7826
7827	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7828	{
7829	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7830	AssertRCReturn(rc, rc);
7831	rc = EMInterpretCLTS(pVM, pVCpu);
7832	AssertRCReturn(rc, rc);
7833	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7834	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7835	Log(("CRX CLTS write rc=%d\n", rc));
7836	break;
7837	}
7838
7839	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7840	{
7841	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7842	AssertRCReturn(rc, rc);
7843	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7844	if (RT_LIKELY(rc == VINF_SUCCESS))
7845	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7846	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7847	Log(("CRX LMSW write rc=%d\n", rc));
7848	break;
7849	}
7850
7851	default:
7852	{
7853	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7854	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7855	}
7856	}
7857
7858	/* Validate possible error codes. */
7859	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7860	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7861	if (RT_SUCCESS(rc))
7862	{
7863	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7864	AssertRCReturn(rc2, rc2);
7865	}
7866
7867	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7874	* VM-exit.
7875	*/
7876	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7877	{
7878	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7879	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7880
7881	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7882	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7883	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7884	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7885	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7886	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7887	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7888	AssertRCReturn(rc, rc);
7889	Log(("CS:RIP=%04x:%#RGv\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
7890
7891	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7892	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7893	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7894	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7895	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7896	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7897	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7898
7899	/* I/O operation lookup arrays. */
7900	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7901	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7902
7903	const uint32_t cbSize = s_aIOSize[uIOWidth];
7904	const uint32_t cbInstr = pVmxTransient->cbInstr;
7905	PVM pVM = pVCpu->CTX_SUFF(pVM);
7906	if (fIOString)
7907	{
7908	/* INS/OUTS - I/O String instruction. */
7909	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7910	/** @todo for now manually disassemble later optimize by getting the fields from
7911	* the VMCS. */
7912	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7913	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7914	* segment prefix info. */
7915	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7916	if (RT_SUCCESS(rc))
7917	{
7918	if (fIOWrite)
7919	{
7920	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7921	(DISCPUMODE)pDis->uAddrMode, cbSize);
7922	rc = VBOXSTRICTRC_VAL(rc2);
7923	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7924	}
7925	else
7926	{
7927	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7928	(DISCPUMODE)pDis->uAddrMode, cbSize);
7929	rc = VBOXSTRICTRC_VAL(rc2);
7930	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7931	}
7932	}
7933	else
7934	{
7935	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7936	rc = VINF_EM_RAW_EMULATE_INSTR;
7937	}
7938	}
7939	else
7940	{
7941	/* IN/OUT - I/O instruction. */
7942	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7943	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7944	if (fIOWrite)
7945	{
7946	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7947	rc = VBOXSTRICTRC_VAL(rc2);
7948	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7949	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7950	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7951	}
7952	else
7953	{
7954	uint32_t u32Result = 0;
7955	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7956	rc = VBOXSTRICTRC_VAL(rc2);
7957	if (IOM_SUCCESS(rc))
7958	{
7959	/* Save result of I/O IN instr. in AL/AX/EAX. */
7960	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7961	}
7962	else if (rc == VINF_IOM_R3_IOPORT_READ)
7963	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7964	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7965	}
7966	}
7967
7968	if (IOM_SUCCESS(rc))
7969	{
7970	pMixedCtx->rip += cbInstr;
7971	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7972	if (RT_LIKELY(rc == VINF_SUCCESS))
7973	{
7974	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7975	AssertRCReturn(rc, rc);
7976
7977	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7978	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7979	{
7980	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7981	for (unsigned i = 0; i < 4; i++)
7982	{
7983	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7984	if ( ( uIOPort >= pMixedCtx->dr[i]
7985	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7986	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7987	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7988	{
7989	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7990	uint64_t uDR6 = ASMGetDR6();
7991
7992	/* Clear all breakpoint status flags and set the one we just hit. */
7993	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7994	uDR6 \|= (uint64_t)RT_BIT(i);
7995
7996	/*
7997	* Note: AMD64 Architecture Programmer's Manual 13.1:
7998	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7999	* be cleared by software after the contents have been read.
8000	*/
8001	ASMSetDR6(uDR6);
8002
8003	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8004	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8005
8006	/* Paranoia. */
8007	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8008	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8009	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8010
8011	/* Resync DR7 */
8012	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8013	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8014
8015	/* Set #DB to be injected into the VM and continue guest execution. */
8016	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8017	break;
8018	}
8019	}
8020	}
8021	}
8022	}
8023
8024	#ifdef DEBUG
8025	if (rc == VINF_IOM_R3_IOPORT_READ)
8026	Assert(!fIOWrite);
8027	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8028	Assert(fIOWrite);
8029	else
8030	{
8031	AssertMsg( RT_FAILURE(rc)
8032	\|\| rc == VINF_SUCCESS
8033	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8034	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8035	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8036	}
8037	#endif
8038
8039	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8040	return rc;
8041	}
8042
8043
8044	/**
8045	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8046	* VM-exit.
8047	*/
8048	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8049	{
8050	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8051
8052	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8053	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8054	AssertRCReturn(rc, rc);
8055	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8056	{
8057	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8058	AssertRCReturn(rc, rc);
8059	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8060	{
8061	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8062	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8063	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8064	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8065	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8066	{
8067	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8068	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8069
8070	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8071	Assert(!pVCpu->hm.s.Event.fPending);
8072	pVCpu->hm.s.Event.fPending = true;
8073	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8074	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8075	AssertRCReturn(rc, rc);
8076	if (fErrorCodeValid)
8077	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8078	else
8079	pVCpu->hm.s.Event.u32ErrCode = 0;
8080	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8081	&& uVector == X86_XCPT_PF)
8082	{
8083	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8084	}
8085	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8086	}
8087	}
8088	}
8089	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8090	* emulation. */
8091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8092	return VERR_EM_INTERPRETER;
8093	}
8094
8095
8096	/**
8097	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8098	*/
8099	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8100	{
8101	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8102	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8103	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8104	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8105	AssertRCReturn(rc, rc);
8106	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8107	return VINF_EM_DBG_STOP;
8108	}
8109
8110
8111	/**
8112	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8113	*/
8114	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8115	{
8116	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8117
8118	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8119	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8120	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8121	return VINF_SUCCESS;
8122	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8123	return rc;
8124
8125	#if 0
8126	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8127	* just sync the whole thing. */
8128	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8129	#else
8130	/* Aggressive state sync. for now. */
8131	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8132	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8133	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8134	#endif
8135	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8136	AssertRCReturn(rc, rc);
8137
8138	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8139	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8140	switch (uAccessType)
8141	{
8142	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8143	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8144	{
8145	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8146	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8147	{
8148	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8149	}
8150
8151	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8152	GCPhys &= PAGE_BASE_GC_MASK;
8153	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8154	PVM pVM = pVCpu->CTX_SUFF(pVM);
8155	Log(("ApicAccess uAccessType=%#x GCPhys=%RGp Off=%#x\n", uAccessType, GCPhys,
8156	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8157
8158	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8159	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8160	CPUMCTX2CORE(pMixedCtx), GCPhys);
8161	rc = VBOXSTRICTRC_VAL(rc2);
8162	Log(("ApicAccess rc=%d\n", rc));
8163	if ( rc == VINF_SUCCESS
8164	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8165	\|\| rc == VERR_PAGE_NOT_PRESENT)
8166	{
8167	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8168	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8169	rc = VINF_SUCCESS;
8170	}
8171	break;
8172	}
8173
8174	default:
8175	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8176	rc = VINF_EM_RAW_EMULATE_INSTR;
8177	break;
8178	}
8179
8180	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8181	return rc;
8182	}
8183
8184
8185	/**
8186	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8187	* VM-exit.
8188	*/
8189	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8190	{
8191	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8192
8193	/* We should -not- get this VM-exit if the guest is debugging. */
8194	if (CPUMIsGuestDebugStateActive(pVCpu))
8195	{
8196	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8197	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8198	}
8199
8200	int rc = VERR_INTERNAL_ERROR_5;
8201	if ( !DBGFIsStepping(pVCpu)
8202	&& !CPUMIsHyperDebugStateActive(pVCpu))
8203	{
8204	/* Don't intercept MOV DRx. */
8205	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8206	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8207	AssertRCReturn(rc, rc);
8208
8209	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8210	PVM pVM = pVCpu->CTX_SUFF(pVM);
8211	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8212	AssertRC(rc);
8213	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8214
8215	#ifdef VBOX_WITH_STATISTICS
8216	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8217	AssertRCReturn(rc, rc);
8218	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8219	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8220	else
8221	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8222	#endif
8223	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8224	return VINF_SUCCESS;
8225	}
8226
8227	/*
8228	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8229	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8230	*/
8231	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8232	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8233	AssertRCReturn(rc, rc);
8234
8235	PVM pVM = pVCpu->CTX_SUFF(pVM);
8236	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8237	{
8238	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8239	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8240	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8241	if (RT_SUCCESS(rc))
8242	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8243	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8244	}
8245	else
8246	{
8247	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8248	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8249	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8250	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8251	}
8252
8253	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8254	if (RT_SUCCESS(rc))
8255	{
8256	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8257	AssertRCReturn(rc2, rc2);
8258	}
8259	return rc;
8260	}
8261
8262
8263	/**
8264	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8265	* Conditional VM-exit.
8266	*/
8267	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8268	{
8269	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8270	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8271
8272	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8273	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8274	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8275	return VINF_SUCCESS;
8276	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8277	return rc;
8278
8279	RTGCPHYS GCPhys = 0;
8280	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8281
8282	#if 0
8283	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8284	#else
8285	/* Aggressive state sync. for now. */
8286	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8287	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8288	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8289	#endif
8290	AssertRCReturn(rc, rc);
8291
8292	/*
8293	* If we succeed, resume guest execution.
8294	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8295	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8296	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8297	* weird case. See @bugref{6043}.
8298	*/
8299	PVM pVM = pVCpu->CTX_SUFF(pVM);
8300	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8301	rc = VBOXSTRICTRC_VAL(rc2);
8302	Log(("EPT misconfig at %#RGv RIP=%#RGv rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8303	if ( rc == VINF_SUCCESS
8304	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8305	\|\| rc == VERR_PAGE_NOT_PRESENT)
8306	{
8307	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8308	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8309	return VINF_SUCCESS;
8310	}
8311	return rc;
8312	}
8313
8314
8315	/**
8316	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8317	* VM-exit.
8318	*/
8319	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8320	{
8321	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8322	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8323
8324	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8325	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8326	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8327	return VINF_SUCCESS;
8328	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8329	return rc;
8330
8331	RTGCPHYS GCPhys = 0;
8332	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8333	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8334	#if 0
8335	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8336	#else
8337	/* Aggressive state sync. for now. */
8338	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8339	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8340	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8341	#endif
8342	AssertRCReturn(rc, rc);
8343
8344	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8345	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8346
8347	RTGCUINT uErrorCode = 0;
8348	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8349	uErrorCode \|= X86_TRAP_PF_ID;
8350	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8351	uErrorCode \|= X86_TRAP_PF_RW;
8352	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8353	uErrorCode \|= X86_TRAP_PF_P;
8354
8355	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8356
8357	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8358	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8359
8360	/* Handle the pagefault trap for the nested shadow table. */
8361	PVM pVM = pVCpu->CTX_SUFF(pVM);
8362	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8363	TRPMResetTrap(pVCpu);
8364
8365	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8366	if ( rc == VINF_SUCCESS
8367	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8368	\|\| rc == VERR_PAGE_NOT_PRESENT)
8369	{
8370	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8371	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8372	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8373	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8374	return VINF_SUCCESS;
8375	}
8376
8377	Log(("EPT return to ring-3 rc=%d\n"));
8378	return rc;
8379	}
8380
8381
8382	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8383	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8384	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8385	/**
8386	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8387	*/
8388	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8389	{
8390	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8391	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8392
8393	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8394	AssertRCReturn(rc, rc);
8395
8396	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8397	{
8398	/* Old-style FPU error reporting needs some extra work. */
8399	/** @todo don't fall back to the recompiler, but do it manually. */
8400	return VERR_EM_INTERPRETER;
8401	}
8402	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8403	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8404	return rc;
8405	}
8406
8407
8408	/**
8409	* VM-exit exception handler for #BP (Breakpoint exception).
8410	*/
8411	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8412	{
8413	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8414	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8415
8416	/** @todo Try optimize this by not saving the entire guest state unless
8417	* really needed. */
8418	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8419	AssertRCReturn(rc, rc);
8420
8421	PVM pVM = pVCpu->CTX_SUFF(pVM);
8422	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8423	if (rc == VINF_EM_RAW_GUEST_TRAP)
8424	{
8425	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8426	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8427	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8428	AssertRCReturn(rc, rc);
8429
8430	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8431	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8432	}
8433
8434	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8435	return rc;
8436	}
8437
8438
8439	/**
8440	* VM-exit exception handler for #DB (Debug exception).
8441	*/
8442	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8443	{
8444	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8445	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8446
8447	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8448	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8449	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8450	AssertRCReturn(rc, rc);
8451
8452	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8453	uint64_t uDR6 = X86_DR6_INIT_VAL;
8454	uDR6 \|= (pVmxTransient->uExitQualification
8455	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8456	PVM pVM = pVCpu->CTX_SUFF(pVM);
8457	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8458	if (rc == VINF_EM_RAW_GUEST_TRAP)
8459	{
8460	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8461	pMixedCtx->dr[6] = uDR6;
8462
8463	if (CPUMIsGuestDebugStateActive(pVCpu))
8464	ASMSetDR6(pMixedCtx->dr[6]);
8465
8466	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8467
8468	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8469	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8470
8471	/* Paranoia. */
8472	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8473	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8474	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8475
8476	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8477	AssertRCReturn(rc,rc);
8478
8479	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8480	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8481	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8482	AssertRCReturn(rc2, rc2);
8483	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8484	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8485	rc = VINF_SUCCESS;
8486	}
8487
8488	return rc;
8489	}
8490
8491
8492	/**
8493	* VM-exit exception handler for #NM (Device-not-available exception: floating
8494	* point exception).
8495	*/
8496	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8497	{
8498	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8499
8500	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8501	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8502	#endif
8503
8504	/* We require CR0 and EFER. EFER is always up-to-date. */
8505	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8506	AssertRCReturn(rc, rc);
8507
8508	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8509	PVM pVM = pVCpu->CTX_SUFF(pVM);
8510	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8511	if (rc == VINF_SUCCESS)
8512	{
8513	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8514	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8515	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8516	return VINF_SUCCESS;
8517	}
8518
8519	/* Forward #NM to the guest. */
8520	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8521	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8522	AssertRCReturn(rc, rc);
8523	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8524	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8525	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8526	return rc;
8527	}
8528
8529
8530	/**
8531	* VM-exit exception handler for #GP (General-protection exception).
8532	*
8533	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8534	*/
8535	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8536	{
8537	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8538	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8539
8540	int rc = VERR_INTERNAL_ERROR_5;
8541	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8542	{
8543	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8544	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8545	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8546	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8547	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8548	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8549	AssertRCReturn(rc, rc);
8550	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RGv CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8551	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8552	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8553	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8554	return rc;
8555	#else
8556	/* We don't intercept #GP. */
8557	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8558	return VERR_VMX_UNEXPECTED_EXCEPTION;
8559	#endif
8560	}
8561
8562	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8563	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8564
8565	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8566	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8567	AssertRCReturn(rc, rc);
8568
8569	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8570	uint32_t cbOp = 0;
8571	PVM pVM = pVCpu->CTX_SUFF(pVM);
8572	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8573	if (RT_SUCCESS(rc))
8574	{
8575	rc = VINF_SUCCESS;
8576	Assert(cbOp == pDis->cbInstr);
8577	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8578	switch (pDis->pCurInstr->uOpcode)
8579	{
8580	case OP_CLI:
8581	pMixedCtx->eflags.Bits.u1IF = 0;
8582	pMixedCtx->rip += pDis->cbInstr;
8583	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8584	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8585	break;
8586
8587	case OP_STI:
8588	pMixedCtx->eflags.Bits.u1IF = 1;
8589	pMixedCtx->rip += pDis->cbInstr;
8590	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8591	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8592	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8593	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8594	break;
8595
8596	case OP_HLT:
8597	rc = VINF_EM_HALT;
8598	pMixedCtx->rip += pDis->cbInstr;
8599	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8600	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8601	break;
8602
8603	case OP_POPF:
8604	{
8605	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8606	uint32_t cbParm = 0;
8607	uint32_t uMask = 0;
8608	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8609	{
8610	cbParm = 4;
8611	uMask = 0xffffffff;
8612	}
8613	else
8614	{
8615	cbParm = 2;
8616	uMask = 0xffff;
8617	}
8618
8619	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8620	RTGCPTR GCPtrStack = 0;
8621	X86EFLAGS uEflags;
8622	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8623	&GCPtrStack);
8624	if (RT_SUCCESS(rc))
8625	{
8626	Assert(sizeof(uEflags.u32) >= cbParm);
8627	uEflags.u32 = 0;
8628	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8629	}
8630	if (RT_FAILURE(rc))
8631	{
8632	rc = VERR_EM_INTERPRETER;
8633	break;
8634	}
8635	Log(("POPF %x -> %#RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8636	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8637	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8638	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8639	pMixedCtx->eflags.Bits.u1RF = 0;
8640	pMixedCtx->esp += cbParm;
8641	pMixedCtx->esp &= uMask;
8642	pMixedCtx->rip += pDis->cbInstr;
8643	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8644	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8645	break;
8646	}
8647
8648	case OP_PUSHF:
8649	{
8650	uint32_t cbParm = 0;
8651	uint32_t uMask = 0;
8652	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8653	{
8654	cbParm = 4;
8655	uMask = 0xffffffff;
8656	}
8657	else
8658	{
8659	cbParm = 2;
8660	uMask = 0xffff;
8661	}
8662
8663	/* Get the stack pointer & push the contents of eflags onto the stack. */
8664	RTGCPTR GCPtrStack = 0;
8665	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8666	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8667	if (RT_FAILURE(rc))
8668	{
8669	rc = VERR_EM_INTERPRETER;
8670	break;
8671	}
8672	X86EFLAGS uEflags;
8673	uEflags = pMixedCtx->eflags;
8674	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8675	uEflags.Bits.u1RF = 0;
8676	uEflags.Bits.u1VM = 0;
8677
8678	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8679	if (RT_FAILURE(rc))
8680	{
8681	rc = VERR_EM_INTERPRETER;
8682	break;
8683	}
8684	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8685	pMixedCtx->esp -= cbParm;
8686	pMixedCtx->esp &= uMask;
8687	pMixedCtx->rip += pDis->cbInstr;
8688	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8690	break;
8691	}
8692
8693	case OP_IRET:
8694	{
8695	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8696	* instruction reference. */
8697	RTGCPTR GCPtrStack = 0;
8698	uint32_t uMask = 0xffff;
8699	uint16_t aIretFrame[3];
8700	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8701	{
8702	rc = VERR_EM_INTERPRETER;
8703	break;
8704	}
8705	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8706	&GCPtrStack);
8707	if (RT_SUCCESS(rc))
8708	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8709	if (RT_FAILURE(rc))
8710	{
8711	rc = VERR_EM_INTERPRETER;
8712	break;
8713	}
8714	pMixedCtx->eip = 0;
8715	pMixedCtx->ip = aIretFrame[0];
8716	pMixedCtx->cs.Sel = aIretFrame[1];
8717	pMixedCtx->cs.ValidSel = aIretFrame[1];
8718	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8719	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8720	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8721	pMixedCtx->sp += sizeof(aIretFrame);
8722	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8723	\| HM_CHANGED_GUEST_RFLAGS;
8724	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8725	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8726	break;
8727	}
8728
8729	case OP_INT:
8730	{
8731	uint16_t uVector = pDis->Param1.uValue & 0xff;
8732	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8733	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8734	break;
8735	}
8736
8737	case OP_INTO:
8738	{
8739	if (pMixedCtx->eflags.Bits.u1OF)
8740	{
8741	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8743	}
8744	break;
8745	}
8746
8747	default:
8748	{
8749	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8750	EMCODETYPE_SUPERVISOR);
8751	rc = VBOXSTRICTRC_VAL(rc2);
8752	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8753	Log(("#GP rc=%Rrc\n", rc));
8754	break;
8755	}
8756	}
8757	}
8758	else
8759	rc = VERR_EM_INTERPRETER;
8760
8761	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8762	("#GP Unexpected rc=%Rrc\n", rc));
8763	return rc;
8764	}
8765
8766
8767	/**
8768	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8769	* the exception reported in the VMX transient structure back into the VM.
8770	*
8771	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8772	* VMX transient structure to be up-to-date.
8773	*/
8774	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8775	{
8776	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8777
8778	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8779	hmR0VmxCheckExitDueToEventDelivery(). */
8780	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8781	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8782	AssertRCReturn(rc, rc);
8783	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8784	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8785	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8786	return VINF_SUCCESS;
8787	}
8788
8789
8790	/**
8791	* VM-exit exception handler for #PF (Page-fault exception).
8792	*/
8793	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8794	{
8795	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8796	PVM pVM = pVCpu->CTX_SUFF(pVM);
8797	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8798	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8799	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8800	AssertRCReturn(rc, rc);
8801
8802	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8803	if (pVM->hm.s.fNestedPaging)
8804	{
8805	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8806	{
8807	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8808	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8809	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8810	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8811	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8812	}
8813	else
8814	{
8815	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8816	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8817	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8818	Log(("Pending #DF due to vectoring #PF. NP\n"));
8819	}
8820	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8821	return rc;
8822	}
8823	#else
8824	Assert(!pVM->hm.s.fNestedPaging);
8825	#endif
8826
8827	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8828	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8829	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8830	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8831	AssertRCReturn(rc, rc);
8832	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8833	if ( pVM->hm.s.fTRPPatchingAllowed
8834	&& pVM->hm.s.pGuestPatchMem
8835	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8836	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8837	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8838	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8839	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8840	{
8841	RTGCPHYS GCPhys;
8842	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8843	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8844	if ( rc == VINF_SUCCESS
8845	&& GCPhys == GCPhysApicBase)
8846	{
8847	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8848	AssertRCReturn(rc, rc);
8849
8850	/* Only attempt to patch the instruction once. */
8851	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8852	if (!pPatch)
8853	return VINF_EM_HM_PATCH_TPR_INSTR;
8854	}
8855	}
8856	#endif
8857
8858	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8859	AssertRCReturn(rc, rc);
8860
8861	Log(("#PF: cr2=%#RGv cs:rip=%#04x:%#RGv uErrCode %#RX32 cr3=%#RGv\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8862	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
8863
8864	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8865	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8866	(RTGCPTR)pVmxTransient->uExitQualification);
8867
8868	Log(("#PF: rc=%Rrc\n", rc));
8869	if (rc == VINF_SUCCESS)
8870	{
8871	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8872	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8873	* memory? We don't update the whole state here... */
8874	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8875	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8876	TRPMResetTrap(pVCpu);
8877	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8878	return rc;
8879	}
8880	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8881	{
8882	if (!pVmxTransient->fVectoringPF)
8883	{
8884	/* It's a guest page fault and needs to be reflected to the guest. */
8885	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8886	TRPMResetTrap(pVCpu);
8887	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8888	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8889	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8890	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
8891	}
8892	else
8893	{
8894	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8895	TRPMResetTrap(pVCpu);
8896	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
8897	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8898	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8899	}
8900
8901	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8902	return VINF_SUCCESS;
8903	}
8904
8905	TRPMResetTrap(pVCpu);
8906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8907	return rc;
8908	}
8909
8910

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45732

Download in other formats: