HMVMXR0.cpp@ 46214

Last change on this file since 46214 was 46192, checked in by vboxsync, 12 years ago
VMM/HM: CR0 and CR4 masks can be 32-bit. Plus hungarian naming fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 381.0 KB

Line
1	/* $Id: HMVMXR0.cpp 46192 2013-05-21 10:25:57Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define HMVMX_SAVE_FULL_GUEST_STATE
39	#define HMVMX_SYNC_FULL_GUEST_STATE
40	#define HMVMX_ALWAYS_TRAP_ALL_XCPTS
41	#define HMVMX_ALWAYS_TRAP_PF
42	#endif
43
44
45	/*******************************************************************************
46	* Defined Constants And Macros *
47	*******************************************************************************/
48	#define HMVMXHCUINTREG RTHCUINTREG
49	#if defined(RT_ARCH_AMD64)
50	# define HMVMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	extern "C" uint32_t g_fVMXIs64bitHost;
53	# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54	# undef HMVMXHCUINTREG
55	# define HMVMXHCUINTREG uint64_t
56	#else
57	# define HMVMX_IS_64BIT_HOST_MODE() (false)
58	#endif
59
60	/** Use the function table. */
61	#define HMVMX_USE_FUNCTION_TABLE
62
63	/** This bit indicates the segment selector is unusable in VT-x. */
64	#define HMVMX_SEL_UNUSABLE RT_BIT(16)
65
66	/** Determine which tagged-TLB flush handler to use. */
67	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
68	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
69	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
70	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
71
72	/** @name Updated-guest-state flags.
73	* @{ */
74	#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
75	#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
76	#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
77	#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
78	#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
79	#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
80	#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
81	#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
82	#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
83	#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
84	#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
85	#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
86	#define HMVMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(12)
87	#define HMVMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(13)
88	#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(14)
89	#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(15)
90	#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(16)
91	#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(17)
92	#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(18)
93	#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
94	#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
95	\| HMVMX_UPDATED_GUEST_RSP \
96	\| HMVMX_UPDATED_GUEST_RFLAGS \
97	\| HMVMX_UPDATED_GUEST_CR0 \
98	\| HMVMX_UPDATED_GUEST_CR3 \
99	\| HMVMX_UPDATED_GUEST_CR4 \
100	\| HMVMX_UPDATED_GUEST_GDTR \
101	\| HMVMX_UPDATED_GUEST_IDTR \
102	\| HMVMX_UPDATED_GUEST_LDTR \
103	\| HMVMX_UPDATED_GUEST_TR \
104	\| HMVMX_UPDATED_GUEST_SEGMENT_REGS \
105	\| HMVMX_UPDATED_GUEST_DEBUG \
106	\| HMVMX_UPDATED_GUEST_FS_BASE_MSR \
107	\| HMVMX_UPDATED_GUEST_GS_BASE_MSR \
108	\| HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
109	\| HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
110	\| HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
111	\| HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
112	\| HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113	\| HMVMX_UPDATED_GUEST_APIC_STATE)
114	/** @} */
115
116	/**
117	* Flags to skip redundant reads of some common VMCS fields that are not part of
118	* the guest-CPU state but are in the transient structure.
119	*/
120	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121	#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122	#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123	#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125	#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126
127	/**
128	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
129	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
130	* we have Nested Paging support.
131	*/
132	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
133	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
134	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
135	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
136	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
137	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
138	\| RT_BIT(X86_XCPT_XF))
139
140	/**
141	* Exception bitmap mask for all contributory exceptions.
142	*/
143	#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
144	\| RT_BIT(X86_XCPT_DE))
145
146	/** Maximum VM-instruction error number. */
147	#define HMVMX_INSTR_ERROR_MAX 28
148
149	/** Profiling macro. */
150	#ifdef HM_PROFILE_EXIT_DISPATCH
151	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
152	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
153	#else
154	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
155	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
156	#endif
157
158
159	/*******************************************************************************
160	* Structures and Typedefs *
161	*******************************************************************************/
162	/**
163	* A state structure for holding miscellaneous information across
164	* VMX non-root operation and restored after the transition.
165	*/
166	typedef struct VMXTRANSIENT
167	{
168	/** The host's rflags/eflags. */
169	RTCCUINTREG uEFlags;
170	#if HC_ARCH_BITS == 32
171	uint32_t u32Alignment0;
172	#endif
173	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
174	uint64_t u64LStarMsr;
175	/** The guest's TPR value used for TPR shadowing. */
176	uint8_t u8GuestTpr;
177	/** Alignment. */
178	uint8_t abAlignment0[6];
179
180	/** The basic VM-exit reason. */
181	uint16_t uExitReason;
182	/** Alignment. */
183	uint16_t u16Alignment0;
184	/** The VM-exit interruption error code. */
185	uint32_t uExitIntrErrorCode;
186	/** The VM-exit exit qualification. */
187	uint64_t uExitQualification;
188
189	/** The VM-exit interruption-information field. */
190	uint32_t uExitIntrInfo;
191	/** The VM-exit instruction-length field. */
192	uint32_t cbInstr;
193	/** Whether the VM-entry failed or not. */
194	bool fVMEntryFailed;
195	/** Alignment. */
196	uint8_t abAlignment1[5];
197
198	/** The VM-entry interruption-information field. */
199	uint32_t uEntryIntrInfo;
200	/** The VM-entry exception error code field. */
201	uint32_t uEntryXcptErrorCode;
202	/** The VM-entry instruction length field. */
203	uint32_t cbEntryInstr;
204
205	/** IDT-vectoring information field. */
206	uint32_t uIdtVectoringInfo;
207	/** IDT-vectoring error code. */
208	uint32_t uIdtVectoringErrorCode;
209
210	/** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_. /
211	uint32_t fVmcsFieldsRead;
212	/** Whether TSC-offsetting should be setup before VM-entry. */
213	bool fUpdateTscOffsettingAndPreemptTimer;
214	/** Whether the VM-exit was caused by a page-fault during delivery of a
215	* contributary exception or a page-fault. */
216	bool fVectoringPF;
217	} VMXTRANSIENT, *PVMXTRANSIENT;
218	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
219	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntrInfo, sizeof(uint64_t));
220	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntrInfo, sizeof(uint64_t));
221
222
223	/**
224	* MSR-bitmap read permissions.
225	*/
226	typedef enum VMXMSREXITREAD
227	{
228	/** Reading this MSR causes a VM-exit. */
229	VMXMSREXIT_INTERCEPT_READ = 0xb,
230	/** Reading this MSR does not cause a VM-exit. */
231	VMXMSREXIT_PASSTHRU_READ
232	} VMXMSREXITREAD;
233
234	/**
235	* MSR-bitmap write permissions.
236	*/
237	typedef enum VMXMSREXITWRITE
238	{
239	/** Writing to this MSR causes a VM-exit. */
240	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
241	/** Writing to this MSR does not cause a VM-exit. */
242	VMXMSREXIT_PASSTHRU_WRITE
243	} VMXMSREXITWRITE;
244
245
246	/*******************************************************************************
247	* Internal Functions *
248	*******************************************************************************/
249	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
250	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
251	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState);
252	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
253	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
254	#endif
255	#ifndef HMVMX_USE_FUNCTION_TABLE
256	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
257	#define HMVMX_EXIT_DECL static int
258	#else
259	#define HMVMX_EXIT_DECL static DECLCALLBACK(int)
260	#endif
261
262	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
274	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
275	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
277	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
278	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
279	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
280	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
281	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
282	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
283	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
284	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
285	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
286	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
287	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
289	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
290	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
291	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
292	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
293	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
294	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
295	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
296	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
297	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
298	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
299	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
300	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
301	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
302	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
303	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
304	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
305	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
306
307	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
308	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
309	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
310	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
311	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
312	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
313	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
314
315
316	/*******************************************************************************
317	* Global Variables *
318	*******************************************************************************/
319	#ifdef HMVMX_USE_FUNCTION_TABLE
320	/**
321	* VM-exit handler.
322	*
323	* @returns VBox status code.
324	* @param pVCpu Pointer to the VMCPU.
325	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
326	* out-of-sync. Make sure to update the required
327	* fields before using them.
328	* @param pVmxTransient Pointer to the VMX-transient structure.
329	*/
330	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331	/** Pointer to VM-exit handler. */
332	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
333
334	/**
335	* VMX_EXIT dispatch table.
336	*/
337	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
338	{
339	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
340	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
341	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
342	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
343	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
344	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
345	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
346	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
347	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
348	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
349	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
350	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
351	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
352	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
353	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
354	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
355	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
356	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
357	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
358	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
359	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
360	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
361	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
362	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
363	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
364	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
365	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
366	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
367	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
368	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
369	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
370	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
371	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
372	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
373	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
374	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
375	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
376	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
377	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
378	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
379	/* 40 UNDEFINED */ hmR0VmxExitPause,
380	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
381	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
382	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
383	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
384	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
385	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
386	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
387	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
388	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
389	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
390	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
391	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
392	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
393	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
394	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
395	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
396	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
397	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
398	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
399	};
400	#endif /* HMVMX_USE_FUNCTION_TABLE */
401
402	#ifdef VBOX_STRICT
403	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
404	{
405	/* 0 */ "(Not Used)",
406	/* 1 */ "VMCALL executed in VMX root operation.",
407	/* 2 */ "VMCLEAR with invalid physical address.",
408	/* 3 */ "VMCLEAR with VMXON pointer.",
409	/* 4 */ "VMLAUNCH with non-clear VMCS.",
410	/* 5 */ "VMRESUME with non-launched VMCS.",
411	/* 6 */ "VMRESUME after VMXOFF",
412	/* 7 */ "VM entry with invalid control fields.",
413	/* 8 */ "VM entry with invalid host state fields.",
414	/* 9 */ "VMPTRLD with invalid physical address.",
415	/* 10 */ "VMPTRLD with VMXON pointer.",
416	/* 11 */ "VMPTRLD with incorrect revision identifier.",
417	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
418	/* 13 */ "VMWRITE to read-only VMCS component.",
419	/* 14 */ "(Not Used)",
420	/* 15 */ "VMXON executed in VMX root operation.",
421	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
422	/* 17 */ "VM entry with non-launched executing VMCS.",
423	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
424	/* 19 */ "VMCALL with non-clear VMCS.",
425	/* 20 */ "VMCALL with invalid VM-exit control fields.",
426	/* 21 */ "(Not Used)",
427	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
428	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
429	/* 24 */ "VMCALL with invalid SMM-monitor features.",
430	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
431	/* 26 */ "VM entry with events blocked by MOV SS.",
432	/* 27 */ "(Not Used)",
433	/* 28 */ "Invalid operand to INVEPT/INVVPID."
434	};
435	#endif /* VBOX_STRICT */
436
437
438
439	/**
440	* Updates the VM's last error record. If there was a VMX instruction error,
441	* reads the error data from the VMCS and updates VCPU's last error record as
442	* well.
443	*
444	* @param pVM Pointer to the VM.
445	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
446	* VERR_VMX_UNABLE_TO_START_VM or
447	* VERR_VMX_INVALID_VMCS_FIELD).
448	* @param rc The error code.
449	*/
450	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
451	{
452	AssertPtr(pVM);
453	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
454	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
455	{
456	AssertPtrReturnVoid(pVCpu);
457	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
458	}
459	pVM->hm.s.lLastError = rc;
460	}
461
462
463	/**
464	* Reads the VM-entry interruption-information field from the VMCS into the VMX
465	* transient structure.
466	*
467	* @returns VBox status code.
468	* @param pVmxTransient Pointer to the VMX transient structure.
469	*
470	* @remarks No-long-jump zone!!!
471	*/
472	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
473	{
474	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
475	AssertRCReturn(rc, rc);
476	return VINF_SUCCESS;
477	}
478
479
480	/**
481	* Reads the VM-entry exception error code field from the VMCS into
482	* the VMX transient structure.
483	*
484	* @returns VBox status code.
485	* @param pVmxTransient Pointer to the VMX transient structure.
486	*
487	* @remarks No-long-jump zone!!!
488	*/
489	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
490	{
491	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
492	AssertRCReturn(rc, rc);
493	return VINF_SUCCESS;
494	}
495
496
497	/**
498	* Reads the VM-entry exception error code field from the VMCS into
499	* the VMX transient structure.
500	*
501	* @returns VBox status code.
502	* @param pVCpu Pointer to the VMCPU.
503	* @param pVmxTransient Pointer to the VMX transient structure.
504	*
505	* @remarks No-long-jump zone!!!
506	*/
507	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
508	{
509	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
510	AssertRCReturn(rc, rc);
511	return VINF_SUCCESS;
512	}
513
514
515	/**
516	* Reads the VM-exit interruption-information field from the VMCS into the VMX
517	* transient structure.
518	*
519	* @returns VBox status code.
520	* @param pVCpu Pointer to the VMCPU.
521	* @param pVmxTransient Pointer to the VMX transient structure.
522	*/
523	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
524	{
525	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
526	{
527	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
528	AssertRCReturn(rc, rc);
529	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
530	}
531	return VINF_SUCCESS;
532	}
533
534
535	/**
536	* Reads the VM-exit interruption error code from the VMCS into the VMX
537	* transient structure.
538	*
539	* @returns VBox status code.
540	* @param pVCpu Pointer to the VMCPU.
541	* @param pVmxTransient Pointer to the VMX transient structure.
542	*/
543	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
544	{
545	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
546	{
547	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
548	AssertRCReturn(rc, rc);
549	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
550	}
551	return VINF_SUCCESS;
552	}
553
554
555	/**
556	* Reads the VM-exit instruction length field from the VMCS into the VMX
557	* transient structure.
558	*
559	* @returns VBox status code.
560	* @param pVCpu Pointer to the VMCPU.
561	* @param pVmxTransient Pointer to the VMX transient structure.
562	*/
563	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
564	{
565	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
566	{
567	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
568	AssertRCReturn(rc, rc);
569	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
570	}
571	return VINF_SUCCESS;
572	}
573
574
575	/**
576	* Reads the exit qualification from the VMCS into the VMX transient structure.
577	*
578	* @returns VBox status code.
579	* @param pVCpu Pointer to the VMCPU.
580	* @param pVmxTransient Pointer to the VMX transient structure.
581	*/
582	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
583	{
584	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
585	{
586	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
587	AssertRCReturn(rc, rc);
588	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
589	}
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Reads the IDT-vectoring information field from the VMCS into the VMX
596	* transient structure.
597	*
598	* @returns VBox status code.
599	* @param pVmxTransient Pointer to the VMX transient structure.
600	*
601	* @remarks No-long-jump zone!!!
602	*/
603	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
604	{
605	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
606	{
607	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
608	AssertRCReturn(rc, rc);
609	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
610	}
611	return VINF_SUCCESS;
612	}
613
614
615	/**
616	* Reads the IDT-vectoring error code from the VMCS into the VMX
617	* transient structure.
618	*
619	* @returns VBox status code.
620	* @param pVmxTransient Pointer to the VMX transient structure.
621	*/
622	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623	{
624	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
625	{
626	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
627	AssertRCReturn(rc, rc);
628	pVmxTransient->fVmcsFieldsRead \|= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
629	}
630	return VINF_SUCCESS;
631	}
632
633
634	/**
635	* Enters VMX root mode operation on the current CPU.
636	*
637	* @returns VBox status code.
638	* @param pVM Pointer to the VM (optional, can be NULL, after
639	* a resume).
640	* @param HCPhysCpuPage Physical address of the VMXON region.
641	* @param pvCpuPage Pointer to the VMXON region.
642	*/
643	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
644	{
645	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
646	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
647	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
648
649	if (pVM)
650	{
651	/* Write the VMCS revision dword to the VMXON region. */
652	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
653	}
654
655	/* Enable the VMX bit in CR4 if necessary. */
656	RTCCUINTREG uCr4 = ASMGetCR4();
657	if (!(uCr4 & X86_CR4_VMXE))
658	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
659
660	/* Enter VMX root mode. */
661	int rc = VMXEnable(HCPhysCpuPage);
662	if (RT_FAILURE(rc))
663	ASMSetCR4(uCr4);
664
665	return rc;
666	}
667
668
669	/**
670	* Exits VMX root mode operation on the current CPU.
671	*
672	* @returns VBox status code.
673	*/
674	static int hmR0VmxLeaveRootMode(void)
675	{
676	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
677
678	/* If we're for some reason not in VMX root mode, then don't leave it. */
679	if (ASMGetCR4() & X86_CR4_VMXE)
680	{
681	/* Exit VMX root mode and clear the VMX bit in CR4 */
682	VMXDisable();
683	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
684	return VINF_SUCCESS;
685	}
686
687	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
688	}
689
690
691	/**
692	* Allocates and maps one physically contiguous page. The allocated page is
693	* zero'd out. (Used by various VT-x structures).
694	*
695	* @returns IPRT status code.
696	* @param pMemObj Pointer to the ring-0 memory object.
697	* @param ppVirt Where to store the virtual address of the
698	* allocation.
699	* @param pPhys Where to store the physical address of the
700	* allocation.
701	*/
702	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
703	{
704	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
705	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
706	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
707
708	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
709	if (RT_FAILURE(rc))
710	return rc;
711	ppVirt = RTR0MemObjAddress(pMemObj);
712	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
713	ASMMemZero32(*ppVirt, PAGE_SIZE);
714	return VINF_SUCCESS;
715	}
716
717
718	/**
719	* Frees and unmaps an allocated physical page.
720	*
721	* @param pMemObj Pointer to the ring-0 memory object.
722	* @param ppVirt Where to re-initialize the virtual address of
723	* allocation as 0.
724	* @param pHCPhys Where to re-initialize the physical address of the
725	* allocation as 0.
726	*/
727	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
728	{
729	AssertPtr(pMemObj);
730	AssertPtr(ppVirt);
731	AssertPtr(pHCPhys);
732	if (*pMemObj != NIL_RTR0MEMOBJ)
733	{
734	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
735	AssertRC(rc);
736	*pMemObj = NIL_RTR0MEMOBJ;
737	*ppVirt = 0;
738	*pHCPhys = 0;
739	}
740	}
741
742
743	/**
744	* Worker function to free VT-x related structures.
745	*
746	* @returns IPRT status code.
747	* @param pVM Pointer to the VM.
748	*/
749	static void hmR0VmxStructsFree(PVM pVM)
750	{
751	for (VMCPUID i = 0; i < pVM->cCpus; i++)
752	{
753	PVMCPU pVCpu = &pVM->aCpus[i];
754	AssertPtr(pVCpu);
755
756	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
757	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
758	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
759	#endif
760
761	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
762	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
763
764	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
765	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
766	}
767
768	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
769	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
770	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
771	#endif
772	}
773
774
775	/**
776	* Worker function to allocate VT-x related VM structures.
777	*
778	* @returns IPRT status code.
779	* @param pVM Pointer to the VM.
780	*/
781	static int hmR0VmxStructsAlloc(PVM pVM)
782	{
783	/*
784	* Initialize members up-front so we can cleanup properly on allocation failure.
785	*/
786	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
787	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
788	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
789	pVM->hm.s.vmx.HCPhys##a_Name = 0;
790
791	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
792	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
793	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
794	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
795
796	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
797	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
798	#endif
799	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
800
801	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
802	for (VMCPUID i = 0; i < pVM->cCpus; i++)
803	{
804	PVMCPU pVCpu = &pVM->aCpus[i];
805	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
806	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
807	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
808	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
809	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
810	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
811	#endif
812	}
813	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
814	#undef VMXLOCAL_INIT_VM_MEMOBJ
815
816	/*
817	* Allocate all the VT-x structures.
818	*/
819	int rc = VINF_SUCCESS;
820	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
821	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
822	if (RT_FAILURE(rc))
823	goto cleanup;
824	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
825	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
826	#endif
827
828	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
829	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
830	{
831	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
832	&pVM->hm.s.vmx.HCPhysApicAccess);
833	if (RT_FAILURE(rc))
834	goto cleanup;
835	}
836
837	/*
838	* Initialize per-VCPU VT-x structures.
839	*/
840	for (VMCPUID i = 0; i < pVM->cCpus; i++)
841	{
842	PVMCPU pVCpu = &pVM->aCpus[i];
843	AssertPtr(pVCpu);
844
845	/* Allocate the VM control structure (VMCS). */
846	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
847	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
848	if (RT_FAILURE(rc))
849	goto cleanup;
850
851	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
852	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
853	{
854	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
855	&pVCpu->hm.s.vmx.HCPhysVirtApic);
856	if (RT_FAILURE(rc))
857	goto cleanup;
858	}
859
860	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
861	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
862	{
863	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
864	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
865	if (RT_FAILURE(rc))
866	goto cleanup;
867	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
868	}
869
870	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
871	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
872	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873	if (RT_FAILURE(rc))
874	goto cleanup;
875
876	/* Allocate the VM-exit MSR-load page for the host MSRs. */
877	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
878	if (RT_FAILURE(rc))
879	goto cleanup;
880	#endif
881	}
882
883	return VINF_SUCCESS;
884
885	cleanup:
886	hmR0VmxStructsFree(pVM);
887	return rc;
888	}
889
890
891	/**
892	* Does global VT-x initialization (called during module initialization).
893	*
894	* @returns VBox status code.
895	*/
896	VMMR0DECL(int) VMXR0GlobalInit(void)
897	{
898	#ifdef HMVMX_USE_FUNCTION_TABLE
899	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
900	# ifdef VBOX_STRICT
901	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
902	Assert(g_apfnVMExitHandlers[i]);
903	# endif
904	#endif
905	return VINF_SUCCESS;
906	}
907
908
909	/**
910	* Does global VT-x termination (called during module termination).
911	*/
912	VMMR0DECL(void) VMXR0GlobalTerm()
913	{
914	/* Nothing to do currently. */
915	}
916
917
918	/**
919	* Sets up and activates VT-x on the current CPU.
920	*
921	* @returns VBox status code.
922	* @param pCpu Pointer to the global CPU info struct.
923	* @param pVM Pointer to the VM (can be NULL after a host resume
924	* operation).
925	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
926	* fEnabledByHost is true).
927	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
928	* @a fEnabledByHost is true).
929	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
930	* enable VT-x/AMD-V on the host.
931	*/
932	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
933	{
934	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
935	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
936
937	if (!fEnabledByHost)
938	{
939	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
940	if (RT_FAILURE(rc))
941	return rc;
942	}
943
944	/*
945	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
946	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
947	* each time while reusing a VPID after hitting the MaxASID limit once.
948	*/
949	if ( pVM
950	&& pVM->hm.s.vmx.fVpid
951	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
952	{
953	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
954	pCpu->fFlushAsidBeforeUse = false;
955	}
956	else
957	pCpu->fFlushAsidBeforeUse = true;
958
959	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
960	++pCpu->cTlbFlushes;
961
962	return VINF_SUCCESS;
963	}
964
965
966	/**
967	* Deactivates VT-x on the current CPU.
968	*
969	* @returns VBox status code.
970	* @param pCpu Pointer to the global CPU info struct.
971	* @param pvCpuPage Pointer to the VMXON region.
972	* @param HCPhysCpuPage Physical address of the VMXON region.
973	*/
974	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
975	{
976	NOREF(pCpu);
977	NOREF(pvCpuPage);
978	NOREF(HCPhysCpuPage);
979
980	return hmR0VmxLeaveRootMode();
981	}
982
983
984	/**
985	* Sets the permission bits for the specified MSR in the MSR bitmap.
986	*
987	* @param pVCpu Pointer to the VMCPU.
988	* @param uMSR The MSR value.
989	* @param enmRead Whether reading this MSR causes a VM-exit.
990	* @param enmWrite Whether writing this MSR causes a VM-exit.
991	*/
992	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
993	{
994	int32_t iBit;
995	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
996
997	/*
998	* Layout:
999	* 0x000 - 0x3ff - Low MSR read bits
1000	* 0x400 - 0x7ff - High MSR read bits
1001	* 0x800 - 0xbff - Low MSR write bits
1002	* 0xc00 - 0xfff - High MSR write bits
1003	*/
1004	if (uMsr <= 0x00001FFF)
1005	iBit = uMsr;
1006	else if ( uMsr >= 0xC0000000
1007	&& uMsr <= 0xC0001FFF)
1008	{
1009	iBit = (uMsr - 0xC0000000);
1010	pbMsrBitmap += 0x400;
1011	}
1012	else
1013	{
1014	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1015	return;
1016	}
1017
1018	Assert(iBit <= 0x1fff);
1019	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1020	ASMBitSet(pbMsrBitmap, iBit);
1021	else
1022	ASMBitClear(pbMsrBitmap, iBit);
1023
1024	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1025	ASMBitSet(pbMsrBitmap + 0x800, iBit);
1026	else
1027	ASMBitClear(pbMsrBitmap + 0x800, iBit);
1028	}
1029
1030
1031	/**
1032	* Flushes the TLB using EPT.
1033	*
1034	* @returns VBox status code.
1035	* @param pVM Pointer to the VM.
1036	* @param pVCpu Pointer to the VMCPU.
1037	* @param enmFlush Type of flush.
1038	*/
1039	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1040	{
1041	AssertPtr(pVM);
1042	Assert(pVM->hm.s.fNestedPaging);
1043
1044	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1045
1046	uint64_t descriptor[2];
1047	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1048	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1049
1050	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1051	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1052	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1053	}
1054
1055
1056	/**
1057	* Flushes the TLB using VPID.
1058	*
1059	* @returns VBox status code.
1060	* @param pVM Pointer to the VM.
1061	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1062	* enmFlush).
1063	* @param enmFlush Type of flush.
1064	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1065	* on @a enmFlush).
1066	*/
1067	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1068	{
1069	AssertPtr(pVM);
1070	Assert(pVM->hm.s.vmx.fVpid);
1071
1072	uint64_t descriptor[2];
1073	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1074	{
1075	descriptor[0] = 0;
1076	descriptor[1] = 0;
1077	}
1078	else
1079	{
1080	AssertPtr(pVCpu);
1081	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1082	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1083	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1084	descriptor[1] = GCPtr;
1085	}
1086
1087	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1088	AssertMsg(rc == VINF_SUCCESS,
1089	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1090	if ( RT_SUCCESS(rc)
1091	&& pVCpu)
1092	{
1093	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1094	}
1095	}
1096
1097
1098	/**
1099	* Invalidates a guest page by guest virtual address. Only relevant for
1100	* EPT/VPID, otherwise there is nothing really to invalidate.
1101	*
1102	* @returns VBox status code.
1103	* @param pVM Pointer to the VM.
1104	* @param pVCpu Pointer to the VMCPU.
1105	* @param GCVirt Guest virtual address of the page to invalidate.
1106	*/
1107	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1108	{
1109	AssertPtr(pVM);
1110	AssertPtr(pVCpu);
1111	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1112
1113	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1114	if (!fFlushPending)
1115	{
1116	/*
1117	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1118	* See @bugref{6043} and @bugref{6177}.
1119	*
1120	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1121	* function maybe called in a loop with individual addresses.
1122	*/
1123	if (pVM->hm.s.vmx.fVpid)
1124	{
1125	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1126	{
1127	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1128	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1129	}
1130	else
1131	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1132	}
1133	else if (pVM->hm.s.fNestedPaging)
1134	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1135	}
1136
1137	return VINF_SUCCESS;
1138	}
1139
1140
1141	/**
1142	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1143	* otherwise there is nothing really to invalidate.
1144	*
1145	* @returns VBox status code.
1146	* @param pVM Pointer to the VM.
1147	* @param pVCpu Pointer to the VMCPU.
1148	* @param GCPhys Guest physical address of the page to invalidate.
1149	*/
1150	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1151	{
1152	LogFlowFunc(("%RGp\n", GCPhys));
1153
1154	/*
1155	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1156	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1157	* This function might be called in a loop.
1158	*/
1159	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1161	return VINF_SUCCESS;
1162	}
1163
1164
1165	/**
1166	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1167	* case where neither EPT nor VPID is supported by the CPU.
1168	*
1169	* @param pVM Pointer to the VM.
1170	* @param pVCpu Pointer to the VMCPU.
1171	*
1172	* @remarks Called with interrupts disabled.
1173	*/
1174	static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1175	{
1176	NOREF(pVM);
1177	AssertPtr(pVCpu);
1178	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1179	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1180
1181	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1182	AssertPtr(pCpu);
1183
1184	pVCpu->hm.s.TlbShootdown.cPages = 0;
1185	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1186	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1187	pVCpu->hm.s.fForceTLBFlush = false;
1188	return;
1189	}
1190
1191
1192	/**
1193	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1194	*
1195	* @param pVM Pointer to the VM.
1196	* @param pVCpu Pointer to the VMCPU.
1197	* @remarks All references to "ASID" in this function pertains to "VPID" in
1198	* Intel's nomenclature. The reason is, to avoid confusion in compare
1199	* statements since the host-CPU copies are named "ASID".
1200	*
1201	* @remarks Called with interrupts disabled.
1202	*/
1203	static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1204	{
1205	AssertPtr(pVM);
1206	AssertPtr(pVCpu);
1207	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1208	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1209	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1210
1211	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1212	AssertPtr(pCpu);
1213
1214	/*
1215	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1216	* This can happen both for start & resume due to long jumps back to ring-3.
1217	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1218	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1219	*/
1220	bool fNewASID = false;
1221	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1222	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1223	{
1224	pVCpu->hm.s.fForceTLBFlush = true;
1225	fNewASID = true;
1226	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1227	}
1228
1229	/*
1230	* Check for explicit TLB shootdowns.
1231	*/
1232	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1233	{
1234	pVCpu->hm.s.fForceTLBFlush = true;
1235	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1236	}
1237
1238	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1239	if (pVCpu->hm.s.fForceTLBFlush)
1240	{
1241	if (fNewASID)
1242	{
1243	++pCpu->uCurrentAsid;
1244	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1245	{
1246	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1247	pCpu->cTlbFlushes++;
1248	pCpu->fFlushAsidBeforeUse = true;
1249	}
1250
1251	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1252	if (pCpu->fFlushAsidBeforeUse)
1253	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1254	}
1255	else
1256	{
1257	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1258	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1259	else
1260	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1261	}
1262
1263	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1264	pVCpu->hm.s.fForceTLBFlush = false;
1265	}
1266	else
1267	{
1268	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1269	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1270	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1271	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1272
1273	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1274	* not be executed. See hmQueueInvlPage() where it is commented
1275	* out. Support individual entry flushing someday. */
1276	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1277	{
1278	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1279
1280	/*
1281	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1282	* as supported by the CPU.
1283	*/
1284	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1285	{
1286	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1287	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1288	}
1289	else
1290	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1291	}
1292	else
1293	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1294	}
1295	pVCpu->hm.s.TlbShootdown.cPages = 0;
1296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1297
1298	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1299	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1300	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1301	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1302	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1303	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1304
1305	/* Update VMCS with the VPID. */
1306	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1307	AssertRC(rc);
1308	}
1309
1310
1311	/**
1312	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1313	*
1314	* @returns VBox status code.
1315	* @param pVM Pointer to the VM.
1316	* @param pVCpu Pointer to the VMCPU.
1317	*
1318	* @remarks Called with interrupts disabled.
1319	*/
1320	static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1321	{
1322	AssertPtr(pVM);
1323	AssertPtr(pVCpu);
1324	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1325	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1326
1327	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1328	AssertPtr(pCpu);
1329
1330	/*
1331	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1332	* This can happen both for start & resume due to long jumps back to ring-3.
1333	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1334	*/
1335	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1336	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1337	{
1338	pVCpu->hm.s.fForceTLBFlush = true;
1339	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1340	}
1341
1342	/* Check for explicit TLB shootdown flushes. */
1343	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1344	{
1345	pVCpu->hm.s.fForceTLBFlush = true;
1346	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1347	}
1348
1349	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1350	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1351
1352	if (pVCpu->hm.s.fForceTLBFlush)
1353	{
1354	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1355	pVCpu->hm.s.fForceTLBFlush = false;
1356	}
1357	else
1358	{
1359	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1360	* not be executed. See hmQueueInvlPage() where it is commented
1361	* out. Support individual entry flushing someday. */
1362	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1363	{
1364	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1365	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1366	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1367	}
1368	else
1369	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1370	}
1371
1372	pVCpu->hm.s.TlbShootdown.cPages = 0;
1373	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1374	}
1375
1376
1377	/**
1378	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1379	*
1380	* @returns VBox status code.
1381	* @param pVM Pointer to the VM.
1382	* @param pVCpu Pointer to the VMCPU.
1383	*
1384	* @remarks Called with interrupts disabled.
1385	*/
1386	static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1387	{
1388	AssertPtr(pVM);
1389	AssertPtr(pVCpu);
1390	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1391	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1392
1393	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1394
1395	/*
1396	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1397	* This can happen both for start & resume due to long jumps back to ring-3.
1398	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1399	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1400	*/
1401	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1402	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1403	{
1404	pVCpu->hm.s.fForceTLBFlush = true;
1405	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1406	}
1407
1408	/* Check for explicit TLB shootdown flushes. */
1409	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1410	{
1411	/*
1412	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1413	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1414	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1415	*/
1416	pVCpu->hm.s.fForceTLBFlush = true;
1417	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1418	}
1419
1420	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1421	if (pVCpu->hm.s.fForceTLBFlush)
1422	{
1423	++pCpu->uCurrentAsid;
1424	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1425	{
1426	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1427	pCpu->fFlushAsidBeforeUse = true;
1428	pCpu->cTlbFlushes++;
1429	}
1430
1431	pVCpu->hm.s.fForceTLBFlush = false;
1432	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1433	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1434	if (pCpu->fFlushAsidBeforeUse)
1435	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1436	}
1437	else
1438	{
1439	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1440	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1441	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1442	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1443
1444	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1445	* not be executed. See hmQueueInvlPage() where it is commented
1446	* out. Support individual entry flushing someday. */
1447	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1448	{
1449	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1450	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1451	{
1452	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1453	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1454	}
1455	else
1456	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1457	}
1458	else
1459	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1460	}
1461
1462	pVCpu->hm.s.TlbShootdown.cPages = 0;
1463	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1464
1465	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1466	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1467	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1468	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1469	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1470	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1471
1472	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1473	AssertRC(rc);
1474	}
1475
1476
1477	/**
1478	* Flushes the guest TLB entry based on CPU capabilities.
1479	*
1480	* @param pVCpu Pointer to the VMCPU.
1481	*/
1482	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1483	{
1484	PVM pVM = pVCpu->CTX_SUFF(pVM);
1485	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1486	{
1487	case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1488	case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1489	case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1490	case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1491	default:
1492	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1493	break;
1494	}
1495	}
1496
1497
1498	/**
1499	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1500	* TLB entries from the host TLB before VM-entry.
1501	*
1502	* @returns VBox status code.
1503	* @param pVM Pointer to the VM.
1504	*/
1505	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1506	{
1507	/*
1508	* Determine optimal flush type for nested paging.
1509	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1510	* guest execution (see hmR3InitFinalizeR0()).
1511	*/
1512	if (pVM->hm.s.fNestedPaging)
1513	{
1514	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1515	{
1516	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1517	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1518	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1520	else
1521	{
1522	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1523	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1524	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1525	}
1526
1527	/* Make sure the write-back cacheable memory type for EPT is supported. */
1528	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1529	{
1530	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1531	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1532	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1533	}
1534	}
1535	else
1536	{
1537	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1538	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1539	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1540	}
1541	}
1542
1543	/*
1544	* Determine optimal flush type for VPID.
1545	*/
1546	if (pVM->hm.s.vmx.fVpid)
1547	{
1548	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1549	{
1550	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1551	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1552	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1553	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1554	else
1555	{
1556	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1557	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1558	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1559	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1560	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1561	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1562	pVM->hm.s.vmx.fVpid = false;
1563	}
1564	}
1565	else
1566	{
1567	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1568	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1569	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1570	pVM->hm.s.vmx.fVpid = false;
1571	}
1572	}
1573
1574	/*
1575	* Setup the handler for flushing tagged-TLBs.
1576	*/
1577	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1578	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
1579	else if (pVM->hm.s.fNestedPaging)
1580	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
1581	else if (pVM->hm.s.vmx.fVpid)
1582	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
1583	else
1584	pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Sets up pin-based VM-execution controls in the VMCS.
1591	*
1592	* @returns VBox status code.
1593	* @param pVM Pointer to the VM.
1594	* @param pVCpu Pointer to the VMCPU.
1595	*/
1596	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1597	{
1598	AssertPtr(pVM);
1599	AssertPtr(pVCpu);
1600
1601	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1602	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1603
1604	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1605	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1606	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
1607
1608	/* Enable the VMX preemption timer. */
1609	if (pVM->hm.s.vmx.fUsePreemptTimer)
1610	{
1611	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
1612	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
1613	}
1614
1615	if ((val & zap) != val)
1616	{
1617	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1618	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1619	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1620	}
1621
1622	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
1623	AssertRCReturn(rc, rc);
1624
1625	/* Update VCPU with the currently set pin-based VM-execution controls. */
1626	pVCpu->hm.s.vmx.u32PinCtls = val;
1627	return rc;
1628	}
1629
1630
1631	/**
1632	* Sets up processor-based VM-execution controls in the VMCS.
1633	*
1634	* @returns VBox status code.
1635	* @param pVM Pointer to the VM.
1636	* @param pVMCPU Pointer to the VMCPU.
1637	*/
1638	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1639	{
1640	AssertPtr(pVM);
1641	AssertPtr(pVCpu);
1642
1643	int rc = VERR_INTERNAL_ERROR_5;
1644	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1645	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1646
1647	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
1648	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1649	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1651	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1652	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1653	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1654
1655	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1656	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
1657	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
1658	{
1659	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
1660	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1661	}
1662
1663	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1664	if (!pVM->hm.s.fNestedPaging)
1665	{
1666	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1667	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
1668	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
1669	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
1670	}
1671
1672	/* Use TPR shadowing if supported by the CPU. */
1673	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
1674	{
1675	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1676	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1677	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1678	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1679	AssertRCReturn(rc, rc);
1680
1681	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1682	/* CR8 writes causes a VM-exit based on TPR threshold. */
1683	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
1684	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
1685	}
1686	else
1687	{
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1689	\| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1690	}
1691
1692	/* Use MSR-bitmaps if supported by the CPU. */
1693	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1694	{
1695	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
1696
1697	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1698	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1699	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1700	AssertRCReturn(rc, rc);
1701
1702	/*
1703	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1704	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1705	*/
1706	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1707	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1708	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1709	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1710	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1711	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1712	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1713	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1714	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1715	}
1716
1717	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1719	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1720
1721	if ((val & zap) != val)
1722	{
1723	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1724	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1725	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1726	}
1727
1728	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
1729	AssertRCReturn(rc, rc);
1730
1731	/* Update VCPU with the currently set processor-based VM-execution controls. */
1732	pVCpu->hm.s.vmx.u32ProcCtls = val;
1733
1734	/*
1735	* Secondary processor-based VM-execution controls.
1736	*/
1737	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1738	{
1739	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1740	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1741
1742	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1743	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1744
1745	if (pVM->hm.s.fNestedPaging)
1746	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1747	else
1748	{
1749	/*
1750	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1751	* VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
1752	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1753	*/
1754	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1755	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1756	}
1757
1758	if (pVM->hm.s.vmx.fVpid)
1759	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1760
1761	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1762	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1763
1764	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1765	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1766	* done dynamically. */
1767	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1768	{
1769	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1770	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1771	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1772	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1773	AssertRCReturn(rc, rc);
1774	}
1775
1776	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1777	{
1778	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1779	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1780	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1781	}
1782
1783	if ((val & zap) != val)
1784	{
1785	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1786	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1787	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1788	}
1789
1790	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
1791	AssertRCReturn(rc, rc);
1792
1793	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1794	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1795	}
1796
1797	return VINF_SUCCESS;
1798	}
1799
1800
1801	/**
1802	* Sets up miscellaneous (everything other than Pin & Processor-based
1803	* VM-execution) control fields in the VMCS.
1804	*
1805	* @returns VBox status code.
1806	* @param pVM Pointer to the VM.
1807	* @param pVCpu Pointer to the VMCPU.
1808	*/
1809	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1810	{
1811	AssertPtr(pVM);
1812	AssertPtr(pVCpu);
1813
1814	/** @todo Shouldn't we able to avoid initializing with 0? */
1815	int rc = VERR_GENERAL_FAILURE;
1816
1817	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1818	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
1819	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
1820
1821	/*
1822	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1823	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1824	* We thus use the exception bitmap to control it rather than use both.
1825	*/
1826	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
1827	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
1828
1829	/** @todo Explore possibility of using IO-bitmaps. */
1830	/* All IO & IOIO instructions cause VM-exits. */
1831	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
1832	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
1833
1834	/* Initialize the MSR-bitmap area. */
1835	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1836	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
1837	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
1838
1839	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1840	/* Setup MSR autoloading/storing. */
1841	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1842	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1843	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1844	AssertRCReturn(rc, rc);
1845	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1846	AssertRCReturn(rc, rc);
1847
1848	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1849	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1850	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1851	AssertRCReturn(rc, rc);
1852	#endif
1853
1854	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1855	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
1856	AssertRCReturn(rc, rc);
1857
1858	/* Setup debug controls */
1859	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
1860	AssertRCReturn(rc, rc);
1861	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
1862	AssertRCReturn(rc, rc);
1863	return rc;
1864	}
1865
1866
1867	/**
1868	* Sets up the initial exception bitmap in the VMCS based on static conditions
1869	* (i.e. conditions that cannot ever change at runtime).
1870	*
1871	* @returns VBox status code.
1872	* @param pVM Pointer to the VM.
1873	* @param pVCpu Pointer to the VMCPU.
1874	*/
1875	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1876	{
1877	AssertPtr(pVM);
1878	AssertPtr(pVCpu);
1879
1880	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1881
1882	uint32_t u32XcptBitmap = 0;
1883
1884	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1885	if (!pVM->hm.s.fNestedPaging)
1886	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1887
1888	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1889	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1890	AssertRCReturn(rc, rc);
1891	return rc;
1892	}
1893
1894
1895	/**
1896	* Sets up the initial guest-state mask. The guest-state mask is consulted
1897	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1898	* for the nested virtualization case (as it would cause a VM-exit).
1899	*
1900	* @param pVCpu Pointer to the VMCPU.
1901	*/
1902	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1903	{
1904	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1905	pVCpu->hm.s.vmx.fUpdatedGuestState = HMVMX_UPDATED_GUEST_ALL;
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Does per-VM VT-x initialization.
1912	*
1913	* @returns VBox status code.
1914	* @param pVM Pointer to the VM.
1915	*/
1916	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1917	{
1918	LogFlowFunc(("pVM=%p\n", pVM));
1919
1920	int rc = hmR0VmxStructsAlloc(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	return VINF_SUCCESS;
1928	}
1929
1930
1931	/**
1932	* Does per-VM VT-x termination.
1933	*
1934	* @returns VBox status code.
1935	* @param pVM Pointer to the VM.
1936	*/
1937	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1938	{
1939	LogFlowFunc(("pVM=%p\n", pVM));
1940
1941	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1942	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1943	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1944	#endif
1945	hmR0VmxStructsFree(pVM);
1946	return VINF_SUCCESS;
1947	}
1948
1949
1950	/**
1951	* Sets up the VM for execution under VT-x.
1952	* This function is only called once per-VM during initalization.
1953	*
1954	* @returns VBox status code.
1955	* @param pVM Pointer to the VM.
1956	*/
1957	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1958	{
1959	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1960	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1961
1962	LogFlowFunc(("pVM=%p\n", pVM));
1963
1964	/*
1965	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1966	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1967	*/
1968	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1969	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1970	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1971	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1972	{
1973	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1974	return VERR_INTERNAL_ERROR;
1975	}
1976
1977	/* Initialize these always, see hmR3InitFinalizeR0().*/
1978	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1979	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1980
1981	/* Setup the tagged-TLB flush handlers. */
1982	int rc = hmR0VmxSetupTaggedTlb(pVM);
1983	if (RT_FAILURE(rc))
1984	{
1985	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1986	return rc;
1987	}
1988
1989	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1990	{
1991	PVMCPU pVCpu = &pVM->aCpus[i];
1992	AssertPtr(pVCpu);
1993	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1994
1995	/* Set revision dword at the beginning of the VMCS structure. */
1996	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1997
1998	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1999	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2000	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2001	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2002
2003	/* Load this VMCS as the current VMCS. */
2004	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2005	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2007
2008	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2009	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2010	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2011
2012	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2013	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2014	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2015
2016	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2017	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2018	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2019
2020	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2021	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2022	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2023
2024	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2025	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2026	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2027
2028	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2029	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2030	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2031	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2032	#endif
2033
2034	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2035	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2036	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2037	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2038
2039	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2040	}
2041
2042	return VINF_SUCCESS;
2043	}
2044
2045
2046	/**
2047	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2048	* the VMCS.
2049	*
2050	* @returns VBox status code.
2051	* @param pVM Pointer to the VM.
2052	* @param pVCpu Pointer to the VMCPU.
2053	*/
2054	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2055	{
2056	RTCCUINTREG uReg = ASMGetCR0();
2057	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2058	AssertRCReturn(rc, rc);
2059
2060	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2061	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2062	if (HMVMX_IS_64BIT_HOST_MODE())
2063	{
2064	uint64_t uRegCR3 = HMR0Get64bitCR3();
2065	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2066	}
2067	else
2068	#endif
2069	{
2070	uReg = ASMGetCR3();
2071	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2072	}
2073	AssertRCReturn(rc, rc);
2074
2075	uReg = ASMGetCR4();
2076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2077	AssertRCReturn(rc, rc);
2078	return rc;
2079	}
2080
2081
2082	/**
2083	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2084	* the host-state area in the VMCS.
2085	*
2086	* @returns VBox status code.
2087	* @param pVM Pointer to the VM.
2088	* @param pVCpu Pointer to the VMCPU.
2089	*/
2090	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2091	{
2092	int rc = VERR_INTERNAL_ERROR_5;
2093	RTSEL uSelCS = 0;
2094	RTSEL uSelSS = 0;
2095	RTSEL uSelDS = 0;
2096	RTSEL uSelES = 0;
2097	RTSEL uSelFS = 0;
2098	RTSEL uSelGS = 0;
2099	RTSEL uSelTR = 0;
2100
2101	/*
2102	* Host Selector registers.
2103	*/
2104	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2105	if (HMVMX_IS_64BIT_HOST_MODE())
2106	{
2107	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2108	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2109	}
2110	else
2111	{
2112	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2113	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2114	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2115	}
2116	#else
2117	uSelCS = ASMGetCS();
2118	uSelSS = ASMGetSS();
2119	#endif
2120
2121	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2122	uSelTR = ASMGetTR();
2123
2124	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2125	/** @todo Verify if we have any platform that actually run with DS or ES with
2126	* RPL != 0 in kernel space. */
2127	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2128	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2129	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2130	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2131	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2132	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2133	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2134	Assert(uSelCS);
2135	Assert(uSelTR);
2136
2137	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2138	#if 0
2139	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2140	Assert(uSelSS != 0);
2141	#endif
2142
2143	/* Write these host selector fields into the host-state area in the VMCS. */
2144	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2145	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2146	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2147	#if 0
2148	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2149	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2150	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2151	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2152	#endif
2153	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2154
2155	/*
2156	* Host GDTR and IDTR.
2157	*/
2158	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2159	* be safe to -not- save and restore GDTR and IDTR in the assembly
2160	* code and just do it here and don't care if the limits are zapped on
2161	* VM-exit. */
2162	RTGDTR Gdtr;
2163	RT_ZERO(Gdtr);
2164	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2165	if (HMVMX_IS_64BIT_HOST_MODE())
2166	{
2167	X86XDTR64 Gdtr64;
2168	X86XDTR64 Idtr64;
2169	HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2170	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2171	rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2172
2173	Gdtr.cbGdt = Gdtr64.cb;
2174	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2175	}
2176	else
2177	#endif
2178	{
2179	RTIDTR Idtr;
2180	ASMGetGDTR(&Gdtr);
2181	ASMGetIDTR(&Idtr);
2182	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2183	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2184	}
2185
2186	/*
2187	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2188	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2189	*/
2190	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2191	{
2192	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2193	return VERR_VMX_INVALID_HOST_STATE;
2194	}
2195
2196	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2197	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2198	if (HMVMX_IS_64BIT_HOST_MODE())
2199	{
2200	/* We need the 64-bit TR base for hybrid darwin. */
2201	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2202	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2203	}
2204	else
2205	#endif
2206	{
2207	uintptr_t uTRBase;
2208	#if HC_ARCH_BITS == 64
2209	uTRBase = X86DESC64_BASE(pDesc);
2210	#else
2211	uTRBase = X86DESC_BASE(pDesc);
2212	#endif
2213	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2214	}
2215	AssertRCReturn(rc, rc);
2216
2217	/*
2218	* Host FS base and GS base.
2219	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2220	* would take care of the bases. In 64-bit, the MSRs come into play.
2221	*/
2222	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2223	if (HMVMX_IS_64BIT_HOST_MODE())
2224	{
2225	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2226	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2227	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2228	AssertRCReturn(rc, rc);
2229	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2230	AssertRCReturn(rc, rc);
2231	}
2232	#endif
2233	return rc;
2234	}
2235
2236
2237	/**
2238	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2239	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2240	* the host after every successful VM exit.
2241	*
2242	* @returns VBox status code.
2243	* @param pVM Pointer to the VM.
2244	* @param pVCpu Pointer to the VMCPU.
2245	*/
2246	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2247	{
2248	AssertPtr(pVCpu);
2249	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2250
2251	int rc = VINF_SUCCESS;
2252	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2253	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2254	uint32_t cHostMsrs = 0;
2255	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2256
2257	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2258	{
2259	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2260	pHostMsr->u32Reserved = 0;
2261	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2262	if (CPUMIsGuestInLongMode(pVCpu))
2263	{
2264	/* Must match the EFER value in our 64 bits switcher. */
2265	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2266	}
2267	else
2268	# endif
2269	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2270	pHostMsr++; cHostMsrs++;
2271	}
2272
2273	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2274	if (HMVMX_IS_64BIT_HOST_MODE())
2275	{
2276	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2277	pHostMsr->u32Reserved = 0;
2278	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2279	pHostMsr++; cHostMsrs++;
2280	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2281	pHostMsr->u32Reserved = 0;
2282	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64-bit mode syscall rip */
2283	pHostMsr++; cHostMsrs++;
2284	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2285	pHostMsr->u32Reserved = 0;
2286	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2287	pHostMsr++; cHostMsrs++;
2288	pHostMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2289	pHostMsr->u32Reserved = 0;
2290	pHostMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2291	pHostMsr++; cHostMsrs++;
2292	}
2293	# endif
2294
2295	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2296	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2297	{
2298	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2299	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2300	}
2301
2302	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2303	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2304
2305	/*
2306	* Host Sysenter MSRs.
2307	*/
2308	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2309	AssertRCReturn(rc, rc);
2310	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2311	if (HMVMX_IS_64BIT_HOST_MODE())
2312	{
2313	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2314	AssertRCReturn(rc, rc);
2315	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2316	}
2317	else
2318	{
2319	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2320	AssertRCReturn(rc, rc);
2321	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2322	}
2323	# elif HC_ARCH_BITS == 32
2324	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2325	AssertRCReturn(rc, rc);
2326	rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2327	# else
2328	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2329	AssertRCReturn(rc, rc);
2330	rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2331	# endif
2332	AssertRCReturn(rc, rc);
2333
2334	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2335	* hmR0VmxSetupExitCtls() !! */
2336	return rc;
2337	}
2338
2339
2340	/**
2341	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2342	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2343	* controls".
2344	*
2345	* @returns VBox status code.
2346	* @param pVCpu Pointer to the VMCPU.
2347	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2348	* out-of-sync. Make sure to update the required fields
2349	* before using them.
2350	*
2351	* @remarks No-long-jump zone!!!
2352	*/
2353	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2354	{
2355	int rc = VINF_SUCCESS;
2356	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2357	{
2358	PVM pVM = pVCpu->CTX_SUFF(pVM);
2359	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2360	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2361
2362	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2363	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
2364
2365	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2366	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2367	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
2368	else
2369	Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
2370
2371	/*
2372	* The following should not be set (since we're not in SMM mode):
2373	* - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
2374	* - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
2375	*/
2376
2377	/** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
2378	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR,
2379	* VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR */
2380
2381	if ((val & zap) != val)
2382	{
2383	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2384	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2385	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2386	}
2387
2388	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
2389	AssertRCReturn(rc, rc);
2390
2391	/* Update VCPU with the currently set VM-exit controls. */
2392	pVCpu->hm.s.vmx.u32EntryCtls = val;
2393	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2394	}
2395	return rc;
2396	}
2397
2398
2399	/**
2400	* Sets up the VM-exit controls in the VMCS.
2401	*
2402	* @returns VBox status code.
2403	* @param pVM Pointer to the VM.
2404	* @param pVCpu Pointer to the VMCPU.
2405	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2406	* out-of-sync. Make sure to update the required fields
2407	* before using them.
2408	*
2409	* @remarks requires EFER.
2410	*/
2411	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2412	{
2413	int rc = VINF_SUCCESS;
2414	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2415	{
2416	PVM pVM = pVCpu->CTX_SUFF(pVM);
2417	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2418	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2419
2420	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2421	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
2422
2423	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2424	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2425	if (HMVMX_IS_64BIT_HOST_MODE())
2426	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
2427	else
2428	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2429	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2430	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2431	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2432	else
2433	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
2434	#endif
2435
2436	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2437	Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
2438
2439	/** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
2440	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
2441	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR,
2442	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR,
2443	* VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR. */
2444
2445	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
2446	val \|= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
2447
2448	if ((val & zap) != val)
2449	{
2450	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2451	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2452	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2453	}
2454
2455	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
2456	AssertRCReturn(rc, rc);
2457
2458	/* Update VCPU with the currently set VM-exit controls. */
2459	pVCpu->hm.s.vmx.u32ExitCtls = val;
2460	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2461	}
2462	return rc;
2463	}
2464
2465
2466	/**
2467	* Loads the guest APIC and related state.
2468	*
2469	* @returns VBox status code.
2470	* @param pVM Pointer to the VM.
2471	* @param pVCpu Pointer to the VMCPU.
2472	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2473	* out-of-sync. Make sure to update the required fields
2474	* before using them.
2475	*/
2476	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2477	{
2478	int rc = VINF_SUCCESS;
2479	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2480	{
2481	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2482	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2483	{
2484	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2485
2486	bool fPendingIntr = false;
2487	uint8_t u8Tpr = 0;
2488	uint8_t u8PendingIntr = 0;
2489	rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
2490	AssertRCReturn(rc, rc);
2491
2492	/*
2493	* If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
2494	* the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
2495	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2496	* the interrupt when we VM-exit for other reasons.
2497	*/
2498	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2499	uint32_t u32TprThreshold = 0;
2500	if (fPendingIntr)
2501	{
2502	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2503	const uint8_t u8PendingPriority = (u8PendingIntr >> 4);
2504	const uint8_t u8TprPriority = (u8Tpr >> 4) & 7;
2505	if (u8PendingPriority <= u8TprPriority)
2506	u32TprThreshold = u8PendingPriority;
2507	else
2508	u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
2509	}
2510	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2511
2512	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2513	AssertRCReturn(rc, rc);
2514
2515	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2516	if (pVCpu->CTX_SUFF(pVM)->hm.s.fTPRPatchingActive)
2517	{
2518	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* EFER always up-to-date. */
2519	pMixedCtx->msrLSTAR = u8Tpr;
2520	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2521	{
2522	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2523	if (fPendingIntr)
2524	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2525	else
2526	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527	}
2528	}
2529	}
2530
2531	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2532	}
2533	return rc;
2534	}
2535
2536
2537	/**
2538	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2539	*
2540	* @returns
2541	* @param pVCpu Pointer to the VMCPU.
2542	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2543	* out-of-sync. Make sure to update the required fields
2544	* before using them.
2545	*
2546	* @remarks No-long-jump zone!!!
2547	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2548	*/
2549	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2550	{
2551	/*
2552	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2553	* inhibit interrupts or clear any existing interrupt-inhibition.
2554	*/
2555	uint32_t uIntrState = 0;
2556	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2557	{
2558	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2559	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS))
2560	== (HMVMX_UPDATED_GUEST_RIP \| HMVMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2561	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2562	{
2563	/*
2564	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2565	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2566	*/
2567	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2568	}
2569	else if (pMixedCtx->eflags.Bits.u1IF)
2570	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2571	else
2572	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2573	}
2574	return uIntrState;
2575	}
2576
2577
2578	/**
2579	* Loads the guest's interruptibility-state into the guest-state area in the
2580	* VMCS.
2581	*
2582	* @returns VBox status code.
2583	* @param pVCpu Pointer to the VMCPU.
2584	* @param uIntrState The interruptibility-state to set.
2585	*/
2586	static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2587	{
2588	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2589	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2590	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2591	AssertRCReturn(rc, rc);
2592	return rc;
2593	}
2594
2595
2596	/**
2597	* Loads the guest's RIP into the guest-state area in the VMCS.
2598	*
2599	* @returns VBox status code.
2600	* @param pVCpu Pointer to the VMCPU.
2601	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2602	* out-of-sync. Make sure to update the required fields
2603	* before using them.
2604	*
2605	* @remarks No-long-jump zone!!!
2606	*/
2607	static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2608	{
2609	int rc = VINF_SUCCESS;
2610	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2611	{
2612	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2613	AssertRCReturn(rc, rc);
2614	Log(("Load: VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2615	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2616	}
2617	return rc;
2618	}
2619
2620
2621	/**
2622	* Loads the guest's RSP into the guest-state area in the VMCS.
2623	*
2624	* @returns VBox status code.
2625	* @param pVCpu Pointer to the VMCPU.
2626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2627	* out-of-sync. Make sure to update the required fields
2628	* before using them.
2629	*
2630	* @remarks No-long-jump zone!!!
2631	*/
2632	static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2633	{
2634	int rc = VINF_SUCCESS;
2635	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2636	{
2637	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2638	AssertRCReturn(rc, rc);
2639	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2640	}
2641	return rc;
2642	}
2643
2644
2645	/**
2646	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2647	*
2648	* @returns VBox status code.
2649	* @param pVCpu Pointer to the VMCPU.
2650	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2651	* out-of-sync. Make sure to update the required fields
2652	* before using them.
2653	*
2654	* @remarks No-long-jump zone!!!
2655	*/
2656	static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2657	{
2658	int rc = VINF_SUCCESS;
2659	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2660	{
2661	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2662	Let us assert it as such and use 32-bit VMWRITE. */
2663	Assert(!(pMixedCtx->rflags.u64 >> 32));
2664	X86EFLAGS uEFlags = pMixedCtx->eflags;
2665	uEFlags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2666	uEFlags.u32 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2667
2668	/*
2669	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2670	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2671	*/
2672	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2673	{
2674	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
2675	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
2676	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uEFlags.u32; /* Save the original eflags of the real-mode guest. */
2677	uEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2678	uEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2679	}
2680
2681	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, uEFlags.u32);
2682	AssertRCReturn(rc, rc);
2683
2684	Log(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", uEFlags.u32));
2685	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2686	}
2687	return rc;
2688	}
2689
2690
2691	/**
2692	* Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
2693	*
2694	* @returns VBox status code.
2695	* @param pVCpu Pointer to the VMCPU.
2696	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2697	* out-of-sync. Make sure to update the required fields
2698	* before using them.
2699	*
2700	* @remarks No-long-jump zone!!!
2701	*/
2702	DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2703	{
2704	int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
2705	AssertRCReturn(rc, rc);
2706	rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
2707	AssertRCReturn(rc, rc);
2708	rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
2709	AssertRCReturn(rc, rc);
2710	return rc;
2711	}
2712
2713
2714	/**
2715	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2716	* in the VMCS.
2717	*
2718	* @returns VBox status code.
2719	* @param pVM Pointer to the VM.
2720	* @param pVCpu Pointer to the VMCPU.
2721	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2722	* out-of-sync. Make sure to update the required fields
2723	* before using them.
2724	*
2725	* @remarks No-long-jump zone!!!
2726	*/
2727	static int hmR0VmxLoadGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pCtx)
2728	{
2729	int rc = VINF_SUCCESS;
2730	PVM pVM = pVCpu->CTX_SUFF(pVM);
2731
2732	/*
2733	* Guest CR0.
2734	* Guest FPU.
2735	*/
2736	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2737	{
2738	Assert(!(pCtx->cr0 >> 32));
2739	uint32_t u32GuestCR0 = pCtx->cr0;
2740
2741	/* The guest's view (read access) of its CR0 is unblemished. */
2742	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
2743	AssertRCReturn(rc, rc);
2744	Log(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
2745
2746	/* Setup VT-x's view of the guest CR0. */
2747	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2748	if (pVM->hm.s.fNestedPaging)
2749	{
2750	if (CPUMIsGuestPagingEnabledEx(pCtx))
2751	{
2752	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2753	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2754	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2755	}
2756	else
2757	{
2758	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2759	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2760	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2761	}
2762
2763	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
2764	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2765	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2766
2767	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2768	AssertRCReturn(rc, rc);
2769	}
2770	else
2771	u32GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2772
2773	/*
2774	* Guest FPU bits.
2775	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2776	* CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
2777	*/
2778	u32GuestCR0 \|= X86_CR0_NE;
2779	bool fInterceptNM = false;
2780	if (CPUMIsGuestFPUStateActive(pVCpu))
2781	{
2782	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2783	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2784	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2785	}
2786	else
2787	{
2788	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2789	u32GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2790	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2791	}
2792
2793	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2794	bool fInterceptMF = false;
2795	if (!(pCtx->cr0 & X86_CR0_NE))
2796	fInterceptMF = true;
2797
2798	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2799	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2800	{
2801	Assert(PDMVmmDevHeapIsEnabled(pVM));
2802	Assert(pVM->hm.s.vmx.pRealModeTSS);
2803	pVCpu->hm.s.vmx.u32XcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
2804	fInterceptNM = true;
2805	fInterceptMF = true;
2806	}
2807	else
2808	pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
2809
2810	if (fInterceptNM)
2811	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2812	else
2813	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2814
2815	if (fInterceptMF)
2816	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2817	else
2818	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2819
2820	/* Additional intercepts for debugging, define these yourself explicitly. */
2821	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
2822	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2823	\| RT_BIT(X86_XCPT_DB)
2824	\| RT_BIT(X86_XCPT_DE)
2825	\| RT_BIT(X86_XCPT_NM)
2826	\| RT_BIT(X86_XCPT_UD)
2827	\| RT_BIT(X86_XCPT_NP)
2828	\| RT_BIT(X86_XCPT_SS)
2829	\| RT_BIT(X86_XCPT_GP)
2830	\| RT_BIT(X86_XCPT_PF)
2831	\| RT_BIT(X86_XCPT_MF);
2832	#elif defined(HMVMX_ALWAYS_TRAP_PF)
2833	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
2834	#endif
2835
2836	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2837
2838	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2839	uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2840	uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2841	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2842	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2843	else
2844	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2845
2846	u32GuestCR0 \|= uSetCR0;
2847	u32GuestCR0 &= uZapCR0;
2848	u32GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2849
2850	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2851	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
2852	AssertRCReturn(rc, rc);
2853	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2854	AssertRCReturn(rc, rc);
2855	Log(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
2856
2857	/*
2858	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2859	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2860	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2861	*/
2862	uint32_t u32CR0Mask = 0;
2863	u32CR0Mask = X86_CR0_PE
2864	\| X86_CR0_NE
2865	\| X86_CR0_WP
2866	\| X86_CR0_PG
2867	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2868	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2869	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2870	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2871	u32CR0Mask &= ~X86_CR0_PE;
2872	if (pVM->hm.s.fNestedPaging)
2873	u32CR0Mask &= ~X86_CR0_WP;
2874
2875	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2876	if (fInterceptNM)
2877	u32CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2878	else
2879	u32CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2880
2881	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2882	pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
2883	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
2884	AssertRCReturn(rc, rc);
2885
2886	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2887	}
2888
2889	/*
2890	* Guest CR2.
2891	* It's always loaded in the assembler code. Nothing to do here.
2892	*/
2893
2894	/*
2895	* Guest CR3.
2896	*/
2897	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2898	{
2899	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
2900	if (pVM->hm.s.fNestedPaging)
2901	{
2902	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2903
2904	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2905	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2906	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
2907	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2908
2909	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2910	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2911	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2912
2913	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2914	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2915	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2916	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2917
2918	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2919	AssertRCReturn(rc, rc);
2920	Log(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2921
2922	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2923	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2924	{
2925	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2926	if (CPUMIsGuestInPAEModeEx(pCtx))
2927	{
2928	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
2929	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
2930	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
2931	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
2932	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
2933	}
2934
2935	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2936	have Unrestricted Execution to handle the guest when it's not using paging. */
2937	GCPhysGuestCR3 = pCtx->cr3;
2938	}
2939	else
2940	{
2941	/*
2942	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2943	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2944	* EPT takes care of translating it to host-physical addresses.
2945	*/
2946	RTGCPHYS GCPhys;
2947	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2948	Assert(PDMVmmDevHeapIsEnabled(pVM));
2949
2950	/* We obtain it here every time as the guest could have relocated this PCI region. */
2951	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2952	AssertRCReturn(rc, rc);
2953
2954	GCPhysGuestCR3 = GCPhys;
2955	}
2956
2957	Log(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
2958	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
2959	}
2960	else
2961	{
2962	/* Non-nested paging case, just use the hypervisor's CR3. */
2963	RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
2964
2965	Log(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
2966	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
2967	}
2968	AssertRCReturn(rc, rc);
2969
2970	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2971	}
2972
2973	/*
2974	* Guest CR4.
2975	*/
2976	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2977	{
2978	Assert(!(pCtx->cr4 >> 32));
2979	uint32_t u32GuestCR4 = pCtx->cr4;
2980
2981	/* The guest's view of its CR4 is unblemished. */
2982	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
2983	AssertRCReturn(rc, rc);
2984	Log(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
2985
2986	/* Setup VT-x's view of the guest CR4. */
2987	/*
2988	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2989	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2990	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2991	*/
2992	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2993	{
2994	Assert(pVM->hm.s.vmx.pRealModeTSS);
2995	Assert(PDMVmmDevHeapIsEnabled(pVM));
2996	u32GuestCR4 &= ~X86_CR4_VME;
2997	}
2998
2999	if (pVM->hm.s.fNestedPaging)
3000	{
3001	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3002	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
3003	{
3004	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3005	u32GuestCR4 \|= X86_CR4_PSE;
3006	/* Our identity mapping is a 32 bits page directory. */
3007	u32GuestCR4 &= ~X86_CR4_PAE;
3008	}
3009	/* else use guest CR4.*/
3010	}
3011	else
3012	{
3013	/*
3014	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3015	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3016	*/
3017	switch (pVCpu->hm.s.enmShadowMode)
3018	{
3019	case PGMMODE_REAL: /* Real-mode. */
3020	case PGMMODE_PROTECTED: /* Protected mode without paging. */
3021	case PGMMODE_32_BIT: /* 32-bit paging. */
3022	{
3023	u32GuestCR4 &= ~X86_CR4_PAE;
3024	break;
3025	}
3026
3027	case PGMMODE_PAE: /* PAE paging. */
3028	case PGMMODE_PAE_NX: /* PAE paging with NX. */
3029	{
3030	u32GuestCR4 \|= X86_CR4_PAE;
3031	break;
3032	}
3033
3034	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3035	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3036	#ifdef VBOX_ENABLE_64_BITS_GUESTS
3037	break;
3038	#endif
3039	default:
3040	AssertFailed();
3041	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3042	}
3043	}
3044
3045	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3046	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3047	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
3048	u32GuestCR4 \|= uSetCR4;
3049	u32GuestCR4 &= uZapCR4;
3050
3051	/* Write VT-x's view of the guest CR4 into the VMCS. */
3052	Log(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3053	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3054	AssertRCReturn(rc, rc);
3055
3056	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
3057	uint32_t u32CR4Mask = 0;
3058	u32CR4Mask = X86_CR4_VME
3059	\| X86_CR4_PAE
3060	\| X86_CR4_PGE
3061	\| X86_CR4_PSE
3062	\| X86_CR4_VMXE;
3063	pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
3064	rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
3065	AssertRCReturn(rc, rc);
3066
3067	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
3068	}
3069	return rc;
3070	}
3071
3072
3073	/**
3074	* Loads the guest debug registers into the guest-state area in the VMCS.
3075	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3076	*
3077	* @returns VBox status code.
3078	* @param pVCpu Pointer to the VMCPU.
3079	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3080	* out-of-sync. Make sure to update the required fields
3081	* before using them.
3082	*
3083	* @remarks No-long-jump zone!!!
3084	*/
3085	static int hmR0VmxLoadGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3086	{
3087	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3088	return VINF_SUCCESS;
3089
3090	#ifdef VBOX_STRICT
3091	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3092	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
3093	{
3094	Assert(!(pMixedCtx->dr[7] >> 32)); /* upper 32 bits are reserved (MBZ). */
3095	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3096	Assert((pMixedCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3097	Assert((pMixedCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3098	}
3099	#endif
3100
3101	int rc = VERR_INTERNAL_ERROR_5;
3102	PVM pVM = pVCpu->CTX_SUFF(pVM);
3103	bool fInterceptDB = false;
3104	bool fInterceptMovDRx = false;
3105	if (DBGFIsStepping(pVCpu))
3106	{
3107	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
3108	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
3109	{
3110	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
3111	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3112	AssertRCReturn(rc, rc);
3113	Assert(fInterceptDB == false);
3114	}
3115	else
3116	{
3117	fInterceptDB = true;
3118	pMixedCtx->eflags.u32 \|= X86_EFL_TF;
3119	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RFLAGS;
3120	}
3121	}
3122
3123	if (CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3124	{
3125	if (!CPUMIsHyperDebugStateActive(pVCpu))
3126	{
3127	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3128	AssertRC(rc);
3129	}
3130	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3131	fInterceptMovDRx = true;
3132	}
3133	else if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3134	{
3135	if (!CPUMIsGuestDebugStateActive(pVCpu))
3136	{
3137	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
3138	AssertRC(rc);
3139	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3140	}
3141	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3142	Assert(fInterceptMovDRx == false);
3143	}
3144	else if (!CPUMIsGuestDebugStateActive(pVCpu))
3145	{
3146	/* For the first time we would need to intercept MOV DRx accesses even when the guest debug registers aren't loaded. */
3147	fInterceptMovDRx = true;
3148	}
3149
3150	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3151	if (fInterceptDB)
3152	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3153	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3154	{
3155	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3156	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3157	#endif
3158	}
3159
3160	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3161	if (fInterceptMovDRx)
3162	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3163	else
3164	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
3165
3166	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3167	AssertRCReturn(rc, rc);
3168	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3169	AssertRCReturn(rc, rc);
3170
3171	/* The guest's view of its DR7 is unblemished. Use 32-bit write as upper 32-bits MBZ as asserted above. */
3172	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
3173	AssertRCReturn(rc, rc);
3174
3175	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3176	return rc;
3177	}
3178
3179
3180	#ifdef VBOX_STRICT
3181	/**
3182	* Strict function to validate segment registers.
3183	*
3184	* @remarks Requires CR0.
3185	*/
3186	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3187	{
3188	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3189	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
3190	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
3191
3192	/* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
3193	* only updates the VMCS bits with the unusable bit and doesn't change the guest-context value. */
3194	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3195	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3196	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3197	{
3198	/* Protected mode checks */
3199	/* CS */
3200	Assert(pCtx->cs.Attr.n.u1Present);
3201	Assert(!(pCtx->cs.Attr.u & 0xf00));
3202	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3203	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3204	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3205	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3206	\|\| (pCtx->cs.Attr.n.u1Granularity));
3207	/* CS cannot be loaded with NULL in protected mode. */
3208	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & HMVMX_SEL_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
3209	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3210	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3211	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3212	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3213	else
3214	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3215	/* SS */
3216	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3217	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3218	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0));
3219	if ( !(pCtx->cr0 & X86_CR0_PE)
3220	\|\| pCtx->cs.Attr.n.u4Type == 3)
3221	{
3222	Assert(!pCtx->ss.Attr.n.u2Dpl);
3223	}
3224	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & HMVMX_SEL_UNUSABLE))
3225	{
3226	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3227	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3228	Assert(pCtx->ss.Attr.n.u1Present);
3229	Assert(!(pCtx->ss.Attr.u & 0xf00));
3230	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3231	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3232	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3233	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3234	\|\| (pCtx->ss.Attr.n.u1Granularity));
3235	}
3236	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3237	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & HMVMX_SEL_UNUSABLE))
3238	{
3239	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3240	Assert(pCtx->ds.Attr.n.u1Present);
3241	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3242	Assert(!(pCtx->ds.Attr.u & 0xf00));
3243	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3244	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3245	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3246	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3247	\|\| (pCtx->ds.Attr.n.u1Granularity));
3248	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3249	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3250	}
3251	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & HMVMX_SEL_UNUSABLE))
3252	{
3253	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3254	Assert(pCtx->es.Attr.n.u1Present);
3255	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3256	Assert(!(pCtx->es.Attr.u & 0xf00));
3257	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3258	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3259	\|\| !(pCtx->es.Attr.n.u1Granularity));
3260	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3261	\|\| (pCtx->es.Attr.n.u1Granularity));
3262	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3263	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3264	}
3265	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & HMVMX_SEL_UNUSABLE))
3266	{
3267	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3268	Assert(pCtx->fs.Attr.n.u1Present);
3269	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3270	Assert(!(pCtx->fs.Attr.u & 0xf00));
3271	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3272	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3273	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3274	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3275	\|\| (pCtx->fs.Attr.n.u1Granularity));
3276	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3277	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3278	}
3279	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & HMVMX_SEL_UNUSABLE))
3280	{
3281	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3282	Assert(pCtx->gs.Attr.n.u1Present);
3283	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3284	Assert(!(pCtx->gs.Attr.u & 0xf00));
3285	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3286	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3287	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3288	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3289	\|\| (pCtx->gs.Attr.n.u1Granularity));
3290	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3291	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3292	}
3293	/* 64-bit capable CPUs. */
3294	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3295	Assert(!(pCtx->cs.u64Base >> 32));
3296	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3297	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3298	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3299	# endif
3300	}
3301	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3302	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3303	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3304	{
3305	/* Real and v86 mode checks. */
3306	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3307	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3308	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3309	{
3310	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3311	}
3312	else
3313	{
3314	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3315	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3316	}
3317
3318	/* CS */
3319	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3320	Assert(pCtx->cs.u32Limit == 0xffff);
3321	Assert(u32CSAttr == 0xf3);
3322	/* SS */
3323	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3324	Assert(pCtx->ss.u32Limit == 0xffff);
3325	Assert(u32SSAttr == 0xf3);
3326	/* DS */
3327	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3328	Assert(pCtx->ds.u32Limit == 0xffff);
3329	Assert(u32DSAttr == 0xf3);
3330	/* ES */
3331	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3332	Assert(pCtx->es.u32Limit == 0xffff);
3333	Assert(u32ESAttr == 0xf3);
3334	/* FS */
3335	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3336	Assert(pCtx->fs.u32Limit == 0xffff);
3337	Assert(u32FSAttr == 0xf3);
3338	/* GS */
3339	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3340	Assert(pCtx->gs.u32Limit == 0xffff);
3341	Assert(u32GSAttr == 0xf3);
3342	/* 64-bit capable CPUs. */
3343	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3344	Assert(!(pCtx->cs.u64Base >> 32));
3345	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3346	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3347	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3348	# endif
3349	}
3350	}
3351	#endif /* VBOX_STRICT */
3352
3353
3354	/**
3355	* Writes a guest segment register into the guest-state area in the VMCS.
3356	*
3357	* @returns VBox status code.
3358	* @param pVCpu Pointer to the VMCPU.
3359	* @param idxSel Index of the selector in the VMCS.
3360	* @param idxLimit Index of the segment limit in the VMCS.
3361	* @param idxBase Index of the segment base in the VMCS.
3362	* @param idxAccess Index of the access rights of the segment in the VMCS.
3363	* @param pSelReg Pointer to the segment selector.
3364	* @param pCtx Pointer to the guest-CPU context.
3365	*
3366	* @remarks No-long-jump zone!!!
3367	*/
3368	static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3369	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3370	{
3371	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3372	AssertRCReturn(rc, rc);
3373	rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3374	AssertRCReturn(rc, rc);
3375	rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3376	AssertRCReturn(rc, rc);
3377
3378	uint32_t u32Access = pSelReg->Attr.u;
3379	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3380	{
3381	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3382	u32Access = 0xf3;
3383	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3384	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3385	}
3386	else
3387	{
3388	/*
3389	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3390	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3391	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3392	* loaded in protected-mode have their attribute as 0.
3393	*/
3394	if (!u32Access)
3395	u32Access = HMVMX_SEL_UNUSABLE;
3396	}
3397
3398	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3399	AssertMsg((u32Access & HMVMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3400	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3401
3402	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3403	AssertRCReturn(rc, rc);
3404	return rc;
3405	}
3406
3407
3408	/**
3409	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3410	* into the guest-state area in the VMCS.
3411	*
3412	* @returns VBox status code.
3413	* @param pVM Pointer to the VM.
3414	* @param pVCPU Pointer to the VMCPU.
3415	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3416	* out-of-sync. Make sure to update the required fields
3417	* before using them.
3418	*
3419	* @remarks Requires CR0 (strict builds validation).
3420	* @remarks No-long-jump zone!!!
3421	*/
3422	static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423	{
3424	int rc = VERR_INTERNAL_ERROR_5;
3425	PVM pVM = pVCpu->CTX_SUFF(pVM);
3426
3427	/*
3428	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3429	*/
3430	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3431	{
3432	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3433	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3434	{
3435	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pMixedCtx->cs.Attr.u;
3436	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pMixedCtx->ss.Attr.u;
3437	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pMixedCtx->ds.Attr.u;
3438	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pMixedCtx->es.Attr.u;
3439	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pMixedCtx->fs.Attr.u;
3440	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pMixedCtx->gs.Attr.u;
3441	}
3442
3443	#ifdef VBOX_WITH_REM
3444	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3445	{
3446	Assert(pVM->hm.s.vmx.pRealModeTSS);
3447	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3448	if ( pVCpu->hm.s.vmx.fWasInRealMode
3449	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
3450	{
3451	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3452	in real-mode (e.g. OpenBSD 4.0) */
3453	REMFlushTBs(pVM);
3454	Log(("Load: Switch to protected mode detected!\n"));
3455	pVCpu->hm.s.vmx.fWasInRealMode = false;
3456	}
3457	}
3458	#endif
3459	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3460	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs, pMixedCtx);
3461	AssertRCReturn(rc, rc);
3462	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3463	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss, pMixedCtx);
3464	AssertRCReturn(rc, rc);
3465	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3466	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds, pMixedCtx);
3467	AssertRCReturn(rc, rc);
3468	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3469	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es, pMixedCtx);
3470	AssertRCReturn(rc, rc);
3471	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3472	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs, pMixedCtx);
3473	AssertRCReturn(rc, rc);
3474	rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3475	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs, pMixedCtx);
3476	AssertRCReturn(rc, rc);
3477
3478	Log(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
3479	pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
3480	#ifdef VBOX_STRICT
3481	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
3482	#endif
3483	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3484	}
3485
3486	/*
3487	* Guest TR.
3488	*/
3489	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3490	{
3491	/*
3492	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3493	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3494	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3495	*/
3496	uint16_t u16Sel = 0;
3497	uint32_t u32Limit = 0;
3498	uint64_t u64Base = 0;
3499	uint32_t u32AccessRights = 0;
3500
3501	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3502	{
3503	u16Sel = pMixedCtx->tr.Sel;
3504	u32Limit = pMixedCtx->tr.u32Limit;
3505	u64Base = pMixedCtx->tr.u64Base;
3506	u32AccessRights = pMixedCtx->tr.Attr.u;
3507	}
3508	else
3509	{
3510	Assert(pVM->hm.s.vmx.pRealModeTSS);
3511	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3512
3513	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3514	RTGCPHYS GCPhys;
3515	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3516	AssertRCReturn(rc, rc);
3517
3518	X86DESCATTR DescAttr;
3519	DescAttr.u = 0;
3520	DescAttr.n.u1Present = 1;
3521	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3522
3523	u16Sel = 0;
3524	u32Limit = HM_VTX_TSS_SIZE;
3525	u64Base = GCPhys; /* in real-mode phys = virt. */
3526	u32AccessRights = DescAttr.u;
3527	}
3528
3529	/* Validate. */
3530	Assert(!(u16Sel & RT_BIT(2)));
3531	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3532	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3533	AssertMsg(!(u32AccessRights & HMVMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3534	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3535	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3536	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3537	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3538	Assert( (u32Limit & 0xfff) == 0xfff
3539	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3540	Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
3541	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3542
3543	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
3544	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
3545	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
3546	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
3547
3548	Log(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3549	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3550	}
3551
3552	/*
3553	* Guest GDTR.
3554	*/
3555	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3556	{
3557	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
3558	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
3559
3560	Assert(!(pMixedCtx->gdtr.cbGdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3561	Log(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
3562	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3563	}
3564
3565	/*
3566	* Guest LDTR.
3567	*/
3568	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3569	{
3570	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3571	uint32_t u32Access = 0;
3572	if (!pMixedCtx->ldtr.Attr.u)
3573	u32Access = HMVMX_SEL_UNUSABLE;
3574	else
3575	u32Access = pMixedCtx->ldtr.Attr.u;
3576
3577	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
3578	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
3579	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
3580	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
3581
3582	/* Validate. */
3583	if (!(u32Access & HMVMX_SEL_UNUSABLE))
3584	{
3585	Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3586	Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3587	Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3588	Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3589	Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3590	Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3591	Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
3592	\|\| !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3593	Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
3594	\|\| pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3595	}
3596
3597	Log(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
3598	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3599	}
3600
3601	/*
3602	* Guest IDTR.
3603	*/
3604	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3605	{
3606	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
3607	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
3608
3609	Assert(!(pMixedCtx->idtr.cbIdt & UINT64_C(0xffff0000))); /* Bits 31:16 MBZ. */
3610	Log(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
3611	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3612	}
3613
3614	return VINF_SUCCESS;
3615	}
3616
3617
3618	/**
3619	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3620	* areas. These MSRs will automatically be loaded to the host CPU on every
3621	* successful VM entry and stored from the host CPU on every successful VM exit.
3622	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3623	*
3624	* @returns VBox status code.
3625	* @param pVCpu Pointer to the VMCPU.
3626	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3627	* out-of-sync. Make sure to update the required fields
3628	* before using them.
3629	*
3630	* @remarks No-long-jump zone!!!
3631	*/
3632	static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3633	{
3634	AssertPtr(pVCpu);
3635	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3636
3637	/*
3638	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3639	*/
3640	int rc = VINF_SUCCESS;
3641	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3642	{
3643	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3644	PVM pVM = pVCpu->CTX_SUFF(pVM);
3645	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3646	uint32_t cGuestMsrs = 0;
3647
3648	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3649	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3650	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3651	if (fSupportsNX \|\| fSupportsLongMode)
3652	{
3653	/** @todo support save IA32_EFER, i.e.
3654	* VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, in which case the
3655	* guest EFER need not be part of the VM-entry MSR-load area. */
3656	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3657	pGuestMsr->u32Reserved = 0;
3658	pGuestMsr->u64Value = pMixedCtx->msrEFER;
3659	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3660	if (!CPUMIsGuestInLongModeEx(pMixedCtx))
3661	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3662	pGuestMsr++; cGuestMsrs++;
3663	if (fSupportsLongMode)
3664	{
3665	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3666	pGuestMsr->u32Reserved = 0;
3667	pGuestMsr->u64Value = pMixedCtx->msrLSTAR; /* 64 bits mode syscall rip */
3668	pGuestMsr++; cGuestMsrs++;
3669	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3670	pGuestMsr->u32Reserved = 0;
3671	pGuestMsr->u64Value = pMixedCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3672	pGuestMsr++; cGuestMsrs++;
3673	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3674	pGuestMsr->u32Reserved = 0;
3675	pGuestMsr->u64Value = pMixedCtx->msrSFMASK; /* syscall flag mask */
3676	pGuestMsr++; cGuestMsrs++;
3677	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3678	pGuestMsr->u32Reserved = 0;
3679	pGuestMsr->u64Value = pMixedCtx->msrKERNELGSBASE; /* swapgs exchange value */
3680	pGuestMsr++; cGuestMsrs++;
3681	}
3682	}
3683
3684	/*
3685	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3686	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3687	*/
3688	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3689	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3690	{
3691	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3692	pGuestMsr->u32Reserved = 0;
3693	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3694	AssertRCReturn(rc, rc);
3695	pGuestMsr++; cGuestMsrs++;
3696	}
3697
3698	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3699	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3700	{
3701	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3702	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3703	}
3704
3705	/* Update the VCPU's copy of the guest MSR count. */
3706	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3707	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3708	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs); AssertRCReturn(rc, rc);
3709	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
3710
3711	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3712	}
3713
3714	/*
3715	* Guest Sysenter MSRs.
3716	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3717	* VM-exits on WRMSRs for these MSRs.
3718	*/
3719	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3720	{
3721	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
3722	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3723	}
3724	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3725	{
3726	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
3727	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3728	}
3729	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3730	{
3731	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
3732	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3733	}
3734
3735	return rc;
3736	}
3737
3738
3739	/**
3740	* Loads the guest activity state into the guest-state area in the VMCS.
3741	*
3742	* @returns VBox status code.
3743	* @param pVCpu Pointer to the VMCPU.
3744	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3745	* out-of-sync. Make sure to update the required fields
3746	* before using them.
3747	*
3748	* @remarks No-long-jump zone!!!
3749	*/
3750	static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
3751	{
3752	/** @todo See if we can make use of other states, e.g.
3753	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3754	int rc = VINF_SUCCESS;
3755	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3756	{
3757	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3758	AssertRCReturn(rc, rc);
3759	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3760	}
3761	return rc;
3762	}
3763
3764
3765	/**
3766	* Sets up the appropriate function to run guest code.
3767	*
3768	* @returns VBox status code.
3769	* @param pVCpu Pointer to the VMCPU.
3770	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
3771	* out-of-sync. Make sure to update the required fields
3772	* before using them.
3773	*
3774	* @remarks No-long-jump zone!!!
3775	*/
3776	static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3777	{
3778	if (CPUMIsGuestInLongModeEx(pMixedCtx))
3779	{
3780	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3781	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3782	#endif
3783	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3784	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3785	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3786	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3787	#else
3788	/* 64-bit host or hybrid host. */
3789	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3790	#endif
3791	}
3792	else
3793	{
3794	/* Guest is not in long mode, use the 32-bit handler. */
3795	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3796	}
3797	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3798	return VINF_SUCCESS;
3799	}
3800
3801
3802	/**
3803	* Wrapper for running the guest code in VT-x.
3804	*
3805	* @returns VBox strict status code.
3806	* @param pVM Pointer to the VM.
3807	* @param pVCpu Pointer to the VMCPU.
3808	* @param pCtx Pointer to the guest-CPU context.
3809	*
3810	* @remarks No-long-jump zone!!!
3811	*/
3812	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3813	{
3814	/*
3815	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3816	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3817	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3818	*/
3819	#ifdef VBOX_WITH_KERNEL_USING_XMM
3820	return HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3821	#else
3822	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3823	#endif
3824	}
3825
3826
3827	/**
3828	* Report world-switch error and dump some useful debug info.
3829	*
3830	* @param pVM Pointer to the VM.
3831	* @param pVCpu Pointer to the VMCPU.
3832	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3833	* @param pCtx Pointer to the guest-CPU context.
3834	* @param pVmxTransient Pointer to the VMX transient structure (only
3835	* exitReason updated).
3836	*/
3837	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3838	{
3839	Assert(pVM);
3840	Assert(pVCpu);
3841	Assert(pCtx);
3842	Assert(pVmxTransient);
3843	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3844
3845	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3846	switch (rcVMRun)
3847	{
3848	case VERR_VMX_INVALID_VMXON_PTR:
3849	AssertFailed();
3850	break;
3851	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3852	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3853	{
3854	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3855	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3856	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3857	AssertRC(rc);
3858
3859	#ifdef VBOX_STRICT
3860	Log(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3861	pVmxTransient->uExitReason));
3862	Log(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
3863	Log(("InstrError %#RX32\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3864	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
3865	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3866	else
3867	Log(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
3868
3869	/* VMX control bits. */
3870	uint32_t u32Val;
3871	uint64_t u64Val;
3872	HMVMXHCUINTREG uHCReg;
3873	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
3874	Log(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
3875	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
3876	Log(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
3877	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
3878	Log(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
3879	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
3880	Log(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
3881	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
3882	Log(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
3883	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3884	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3885	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3886	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3887	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3888	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3889	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3890	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3891	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3892	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3893	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3894	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3895	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3896	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3897	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3898	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3899	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3900	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3901	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3902	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3903	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3904	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3905	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3906	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3907	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3908	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3909	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3910	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3911	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3912	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3913	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3914	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3915
3916	/* Guest bits. */
3917	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
3918	Log(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
3919	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
3920	Log(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
3921	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
3922	Log(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
3923	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3924	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3925
3926	/* Host bits. */
3927	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3928	Log(("Host CR0 %#RHr\n", uHCReg));
3929	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3930	Log(("Host CR3 %#RHr\n", uHCReg));
3931	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3932	Log(("Host CR4 %#RHr\n", uHCReg));
3933
3934	RTGDTR HostGdtr;
3935	PCX86DESCHC pDesc;
3936	ASMGetGDTR(&HostGdtr);
3937	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
3938	Log(("Host CS %#08x\n", u32Val));
3939	if (u32Val < HostGdtr.cbGdt)
3940	{
3941	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3942	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3943	}
3944
3945	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3946	Log(("Host DS %#08x\n", u32Val));
3947	if (u32Val < HostGdtr.cbGdt)
3948	{
3949	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3950	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3951	}
3952
3953	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3954	Log(("Host ES %#08x\n", u32Val));
3955	if (u32Val < HostGdtr.cbGdt)
3956	{
3957	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3958	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3959	}
3960
3961	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3962	Log(("Host FS %#08x\n", u32Val));
3963	if (u32Val < HostGdtr.cbGdt)
3964	{
3965	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3966	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3967	}
3968
3969	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3970	Log(("Host GS %#08x\n", u32Val));
3971	if (u32Val < HostGdtr.cbGdt)
3972	{
3973	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3974	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3975	}
3976
3977	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3978	Log(("Host SS %#08x\n", u32Val));
3979	if (u32Val < HostGdtr.cbGdt)
3980	{
3981	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3982	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3983	}
3984
3985	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3986	Log(("Host TR %#08x\n", u32Val));
3987	if (u32Val < HostGdtr.cbGdt)
3988	{
3989	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3990	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3991	}
3992
3993	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3994	Log(("Host TR Base %#RHv\n", uHCReg));
3995	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3996	Log(("Host GDTR Base %#RHv\n", uHCReg));
3997	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3998	Log(("Host IDTR Base %#RHv\n", uHCReg));
3999	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
4000	Log(("Host SYSENTER CS %#08x\n", u32Val));
4001	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
4002	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
4003	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
4004	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
4005	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
4006	Log(("Host RSP %#RHv\n", uHCReg));
4007	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
4008	Log(("Host RIP %#RHv\n", uHCReg));
4009	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4010	if (HMVMX_IS_64BIT_HOST_MODE())
4011	{
4012	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
4013	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
4014	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4015	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4016	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4017	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
4018	}
4019	# endif
4020	#endif /* VBOX_STRICT */
4021	break;
4022	}
4023
4024	default:
4025	/* Impossible */
4026	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
4027	break;
4028	}
4029	NOREF(pVM);
4030	}
4031
4032
4033	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4034	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
4035	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
4036	#endif
4037	#ifdef VBOX_STRICT
4038	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4039	{
4040	switch (idxField)
4041	{
4042	case VMX_VMCS_GUEST_RIP:
4043	case VMX_VMCS_GUEST_RSP:
4044	case VMX_VMCS_GUEST_SYSENTER_EIP:
4045	case VMX_VMCS_GUEST_SYSENTER_ESP:
4046	case VMX_VMCS_GUEST_GDTR_BASE:
4047	case VMX_VMCS_GUEST_IDTR_BASE:
4048	case VMX_VMCS_GUEST_CS_BASE:
4049	case VMX_VMCS_GUEST_DS_BASE:
4050	case VMX_VMCS_GUEST_ES_BASE:
4051	case VMX_VMCS_GUEST_FS_BASE:
4052	case VMX_VMCS_GUEST_GS_BASE:
4053	case VMX_VMCS_GUEST_SS_BASE:
4054	case VMX_VMCS_GUEST_LDTR_BASE:
4055	case VMX_VMCS_GUEST_TR_BASE:
4056	case VMX_VMCS_GUEST_CR3:
4057	return true;
4058	}
4059	return false;
4060	}
4061
4062	static bool hmR0VmxIsValidReadField(uint32_t idxField)
4063	{
4064	switch (idxField)
4065	{
4066	/* Read-only fields. */
4067	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4068	return true;
4069	}
4070	/* Remaining readable fields should also be writable. */
4071	return hmR0VmxIsValidWriteField(idxField);
4072	}
4073	#endif /* VBOX_STRICT */
4074
4075
4076	/**
4077	* Executes the specified handler in 64-bit mode.
4078	*
4079	* @returns VBox status code.
4080	* @param pVM Pointer to the VM.
4081	* @param pVCpu Pointer to the VMCPU.
4082	* @param pCtx Pointer to the guest CPU context.
4083	* @param enmOp The operation to perform.
4084	* @param cbParam Number of parameters.
4085	* @param paParam Array of 32-bit parameters.
4086	*/
4087	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
4088	uint32_t *paParam)
4089	{
4090	int rc, rc2;
4091	PHMGLOBLCPUINFO pCpu;
4092	RTHCPHYS HCPhysCpuPage;
4093	RTCCUINTREG uOldEFlags;
4094
4095	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4096	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
4097	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4098	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4099
4100	#ifdef VBOX_STRICT
4101	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4102	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4103
4104	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4105	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4106	#endif
4107
4108	/* Disable interrupts. */
4109	uOldEFlags = ASMIntDisableFlags();
4110
4111	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4112	RTCPUID idHostCpu = RTMpCpuId();
4113	CPUMR0SetLApic(pVM, idHostCpu);
4114	#endif
4115
4116	pCpu = HMR0GetCurrentCpu();
4117	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4118
4119	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4120	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4121
4122	/* Leave VMX Root Mode. */
4123	VMXDisable();
4124
4125	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4126
4127	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4128	CPUMSetHyperEIP(pVCpu, enmOp);
4129	for (int i = (int)cbParam - 1; i >= 0; i--)
4130	CPUMPushHyper(pVCpu, paParam[i]);
4131
4132	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4133
4134	/* Call the switcher. */
4135	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4136	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4137
4138	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
4139	/* Make sure the VMX instructions don't cause #UD faults. */
4140	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4141
4142	/* Re-enter VMX Root Mode */
4143	rc2 = VMXEnable(HCPhysCpuPage);
4144	if (RT_FAILURE(rc2))
4145	{
4146	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4147	ASMSetFlags(uOldEFlags);
4148	return rc2;
4149	}
4150
4151	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4152	AssertRC(rc2);
4153	Assert(!(ASMGetFlags() & X86_EFL_IF));
4154	ASMSetFlags(uOldEFlags);
4155	return rc;
4156	}
4157
4158
4159	/**
4160	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4161	* supporting 64-bit guests.
4162	*
4163	* @returns VBox status code.
4164	* @param fResume Whether to VMLAUNCH or VMRESUME.
4165	* @param pCtx Pointer to the guest-CPU context.
4166	* @param pCache Pointer to the VMCS cache.
4167	* @param pVM Pointer to the VM.
4168	* @param pVCpu Pointer to the VMCPU.
4169	*/
4170	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4171	{
4172	uint32_t aParam[6];
4173	PHMGLOBLCPUINFO pCpu = NULL;
4174	RTHCPHYS HCPhysCpuPage = 0;
4175	int rc = VERR_INTERNAL_ERROR_5;
4176
4177	pCpu = HMR0GetCurrentCpu();
4178	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4179
4180	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4181	pCache->uPos = 1;
4182	pCache->interPD = PGMGetInterPaeCR3(pVM);
4183	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4184	#endif
4185
4186	#ifdef VBOX_STRICT
4187	pCache->TestIn.HCPhysCpuPage = 0;
4188	pCache->TestIn.HCPhysVmcs = 0;
4189	pCache->TestIn.pCache = 0;
4190	pCache->TestOut.HCPhysVmcs = 0;
4191	pCache->TestOut.pCache = 0;
4192	pCache->TestOut.pCtx = 0;
4193	pCache->TestOut.eflags = 0;
4194	#endif
4195
4196	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4197	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4198	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4199	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4200	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4201	aParam[5] = 0;
4202
4203	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4204	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4205	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4206	#endif
4207	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
4208
4209	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4210	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4211	Assert(pCtx->dr[4] == 10);
4212	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4213	#endif
4214
4215	#ifdef VBOX_STRICT
4216	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4217	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4218	pVCpu->hm.s.vmx.HCPhysVmcs));
4219	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4220	pCache->TestOut.HCPhysVmcs));
4221	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4222	pCache->TestOut.pCache));
4223	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4224	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4225	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4226	pCache->TestOut.pCtx));
4227	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4228	#endif
4229	return rc;
4230	}
4231
4232
4233	/**
4234	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4235	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4236	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4237	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4238	*
4239	* @returns VBox status code.
4240	* @param pVM Pointer to the VM.
4241	* @param pVCpu Pointer to the VMCPU.
4242	*/
4243	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4244	{
4245	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4246	{ \
4247	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4248	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4249	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4250	++cReadFields; \
4251	}
4252
4253	AssertPtr(pVM);
4254	AssertPtr(pVCpu);
4255	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4256	uint32_t cReadFields = 0;
4257
4258	/*
4259	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
4260	* and serve to indicate exceptions to the rules.
4261	*/
4262
4263	/* Guest-natural selector base fields. */
4264	#if 0
4265	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
4266	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4267	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4268	#endif
4269	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4270	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4271	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4272	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4273	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4274	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4275	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4276	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4277	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4278	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4281	#if 0
4282	/* Unused natural width guest-state fields. */
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4285	#endif
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4288
4289	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4290	#if 0
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4300	#endif
4301
4302	/* Natural width guest-state fields. */
4303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
4304	#if 0
4305	/* Currently unused field. */
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
4307	#endif
4308
4309	if (pVM->hm.s.fNestedPaging)
4310	{
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4312	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
4313	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
4314	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4315	}
4316	else
4317	{
4318	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
4319	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4320	}
4321
4322	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4323	return VINF_SUCCESS;
4324	}
4325
4326
4327	/**
4328	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4329	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4330	* darwin, running 64-bit guests).
4331	*
4332	* @returns VBox status code.
4333	* @param pVCpu Pointer to the VMCPU.
4334	* @param idxField The VMCS field encoding.
4335	* @param u64Val 16, 32 or 64 bits value.
4336	*/
4337	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4338	{
4339	int rc;
4340	switch (idxField)
4341	{
4342	/*
4343	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4344	*/
4345	/* 64-bit Control fields. */
4346	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4347	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4348	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4349	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4350	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4351	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4352	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4353	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4354	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4355	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4356	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4357	case VMX_VMCS64_CTRL_EPTP_FULL:
4358	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4359	/* 64-bit Guest-state fields. */
4360	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4361	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4362	case VMX_VMCS64_GUEST_PAT_FULL:
4363	case VMX_VMCS64_GUEST_EFER_FULL:
4364	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4365	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4366	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4367	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4368	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4369	/* 64-bit Host-state fields. */
4370	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4371	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4372	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4373	{
4374	rc = VMXWriteVmcs32(idxField, u64Val);
4375	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
4376	break;
4377	}
4378
4379	/*
4380	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4381	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4382	*/
4383	/* Natural-width Guest-state fields. */
4384	case VMX_VMCS_GUEST_CR3:
4385	case VMX_VMCS_GUEST_ES_BASE:
4386	case VMX_VMCS_GUEST_CS_BASE:
4387	case VMX_VMCS_GUEST_SS_BASE:
4388	case VMX_VMCS_GUEST_DS_BASE:
4389	case VMX_VMCS_GUEST_FS_BASE:
4390	case VMX_VMCS_GUEST_GS_BASE:
4391	case VMX_VMCS_GUEST_LDTR_BASE:
4392	case VMX_VMCS_GUEST_TR_BASE:
4393	case VMX_VMCS_GUEST_GDTR_BASE:
4394	case VMX_VMCS_GUEST_IDTR_BASE:
4395	case VMX_VMCS_GUEST_RSP:
4396	case VMX_VMCS_GUEST_RIP:
4397	case VMX_VMCS_GUEST_SYSENTER_ESP:
4398	case VMX_VMCS_GUEST_SYSENTER_EIP:
4399	{
4400	if (!(u64Val >> 32))
4401	{
4402	/* If this field is 64-bit, VT-x will zero out the top bits. */
4403	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4404	}
4405	else
4406	{
4407	/* Assert that only the 32->64 switcher case should ever come here. */
4408	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4409	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4410	}
4411	break;
4412	}
4413
4414	default:
4415	{
4416	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
4417	rc = VERR_INVALID_PARAMETER;
4418	break;
4419	}
4420	}
4421	AssertRCReturn(rc, rc);
4422	return rc;
4423	}
4424
4425
4426	/**
4427	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4428	* hosts (except darwin) for 64-bit guests.
4429	*
4430	* @param pVCpu Pointer to the VMCPU.
4431	* @param idxField The VMCS field encoding.
4432	* @param u64Val 16, 32 or 64 bits value.
4433	*/
4434	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4435	{
4436	AssertPtr(pVCpu);
4437	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4438
4439	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4440	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4441
4442	/* Make sure there are no duplicates. */
4443	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4444	{
4445	if (pCache->Write.aField[i] == idxField)
4446	{
4447	pCache->Write.aFieldVal[i] = u64Val;
4448	return VINF_SUCCESS;
4449	}
4450	}
4451
4452	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4453	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4454	pCache->Write.cValidEntries++;
4455	return VINF_SUCCESS;
4456	}
4457
4458	/* Enable later when the assembly code uses these as callbacks. */
4459	#if 0
4460	/*
4461	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4462	*
4463	* @param pVCpu Pointer to the VMCPU.
4464	* @param pCache Pointer to the VMCS cache.
4465	*
4466	* @remarks No-long-jump zone!!!
4467	*/
4468	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4469	{
4470	AssertPtr(pCache);
4471	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4472	{
4473	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4474	AssertRC(rc);
4475	}
4476	pCache->Write.cValidEntries = 0;
4477	}
4478
4479
4480	/**
4481	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4482	*
4483	* @param pVCpu Pointer to the VMCPU.
4484	* @param pCache Pointer to the VMCS cache.
4485	*
4486	* @remarks No-long-jump zone!!!
4487	*/
4488	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4489	{
4490	AssertPtr(pCache);
4491	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4492	{
4493	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4494	AssertRC(rc);
4495	}
4496	}
4497	#endif
4498	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4499
4500
4501	/**
4502	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4503	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4504	* timer.
4505	*
4506	* @returns VBox status code.
4507	* @param pVCpu Pointer to the VMCPU.
4508	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4509	* out-of-sync. Make sure to update the required fields
4510	* before using them.
4511	* @remarks No-long-jump zone!!!
4512	*/
4513	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4514	{
4515	int rc = VERR_INTERNAL_ERROR_5;
4516	bool fOffsettedTsc = false;
4517	PVM pVM = pVCpu->CTX_SUFF(pVM);
4518	if (pVM->hm.s.vmx.fUsePreemptTimer)
4519	{
4520	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4521
4522	/* Make sure the returned values have sane upper and lower boundaries. */
4523	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4524	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4525	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4526	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4527
4528	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4529	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4530	}
4531	else
4532	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4533
4534	if (fOffsettedTsc)
4535	{
4536	uint64_t u64CurTSC = ASMReadTSC();
4537	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4538	{
4539	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4540	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4541
4542	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4543	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4544	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4545	}
4546	else
4547	{
4548	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4549	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4550	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4551	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4552	}
4553	}
4554	else
4555	{
4556	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4557	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
4558	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4559	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4560	}
4561	}
4562
4563
4564	/**
4565	* Determines if an exception is a contributory exception. Contributory
4566	* exceptions are ones which can cause double-faults. Page-fault is
4567	* intentionally not included here as it's a conditional contributory exception.
4568	*
4569	* @returns true if the exception is contributory, false otherwise.
4570	* @param uVector The exception vector.
4571	*/
4572	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4573	{
4574	switch (uVector)
4575	{
4576	case X86_XCPT_GP:
4577	case X86_XCPT_SS:
4578	case X86_XCPT_NP:
4579	case X86_XCPT_TS:
4580	case X86_XCPT_DE:
4581	return true;
4582	default:
4583	break;
4584	}
4585	return false;
4586	}
4587
4588
4589	/**
4590	* Sets an event as a pending event to be injected into the guest.
4591	*
4592	* @param pVCpu Pointer to the VMCPU.
4593	* @param u32IntrInfo The VM-entry interruption-information field.
4594	* @param cbInstr The VM-entry instruction length in bytes (for software
4595	* interrupts, exceptions and privileged software
4596	* exceptions).
4597	* @param u32ErrCode The VM-entry exception error code.
4598	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4599	* page-fault.
4600	*/
4601	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4602	RTGCUINTPTR GCPtrFaultAddress)
4603	{
4604	Assert(!pVCpu->hm.s.Event.fPending);
4605	pVCpu->hm.s.Event.fPending = true;
4606	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4607	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4608	pVCpu->hm.s.Event.cbInstr = cbInstr;
4609	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4610	}
4611
4612
4613	/**
4614	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4615	*
4616	* @param pVCpu Pointer to the VMCPU.
4617	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4618	* out-of-sync. Make sure to update the required fields
4619	* before using them.
4620	*/
4621	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4622	{
4623	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
4624	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4625	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4626	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4627	}
4628
4629
4630	/**
4631	* Handle a condition that occurred while delivering an event through the guest
4632	* IDT.
4633	*
4634	* @returns VBox status code (informational error codes included).
4635	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4636	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4637	* continue execution of the guest which will delivery the #DF.
4638	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4639	*
4640	* @param pVCpu Pointer to the VMCPU.
4641	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4642	* out-of-sync. Make sure to update the required fields
4643	* before using them.
4644	* @param pVmxTransient Pointer to the VMX transient structure.
4645	*
4646	* @remarks No-long-jump zone!!!
4647	*/
4648	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4649	{
4650	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4651	AssertRC(rc);
4652	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4653	{
4654	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4655	AssertRCReturn(rc, rc);
4656
4657	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4658	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4659	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4660
4661	typedef enum
4662	{
4663	VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
4664	VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
4665	VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
4666	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4667	} VMXREFLECTXCPT;
4668
4669	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4670	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4671	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4672	{
4673	enmReflect = VMXREFLECTXCPT_XCPT;
4674	#ifdef VBOX_STRICT
4675	if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4676	&& uExitVector == X86_XCPT_PF)
4677	{
4678	Log(("IDT: Contributory #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4679	}
4680	#endif
4681	if ( uExitVector == X86_XCPT_PF
4682	&& uIdtVector == X86_XCPT_PF)
4683	{
4684	pVmxTransient->fVectoringPF = true;
4685	Log(("IDT: Vectoring #PF uCR2=%#RX64\n", pMixedCtx->cr2));
4686	}
4687	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
4688	&& hmR0VmxIsContributoryXcpt(uExitVector)
4689	&& ( hmR0VmxIsContributoryXcpt(uIdtVector)
4690	\|\| uIdtVector == X86_XCPT_PF))
4691	{
4692	enmReflect = VMXREFLECTXCPT_DF;
4693	}
4694	else if (uIdtVector == X86_XCPT_DF)
4695	enmReflect = VMXREFLECTXCPT_TF;
4696	}
4697	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4698	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4699	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4700	{
4701	/*
4702	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4703	* (whatever they are) as they reoccur when restarting the instruction.
4704	*/
4705	enmReflect = VMXREFLECTXCPT_XCPT;
4706	}
4707
4708	switch (enmReflect)
4709	{
4710	case VMXREFLECTXCPT_XCPT:
4711	{
4712	uint32_t u32ErrCode = 0;
4713	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4714	{
4715	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4716	AssertRCReturn(rc, rc);
4717	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4718	}
4719
4720	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
4721	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4722	0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
4723	rc = VINF_SUCCESS;
4724	Log(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo,
4725	pVCpu->hm.s.Event.u32ErrCode));
4726	break;
4727	}
4728
4729	case VMXREFLECTXCPT_DF:
4730	{
4731	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4732	rc = VINF_VMX_DOUBLE_FAULT;
4733	Log(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo,
4734	uIdtVector, uExitVector));
4735	break;
4736	}
4737
4738	case VMXREFLECTXCPT_TF:
4739	{
4740	Log(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4741	rc = VINF_EM_RESET;
4742	break;
4743	}
4744
4745	default:
4746	Assert(rc == VINF_SUCCESS);
4747	break;
4748	}
4749	}
4750	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4751	return rc;
4752	}
4753
4754
4755	/**
4756	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4757	*
4758	* @returns VBox status code.
4759	* @param pVCpu Pointer to the VMCPU.
4760	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4761	* out-of-sync. Make sure to update the required fields
4762	* before using them.
4763	*
4764	* @remarks No-long-jump zone!!!
4765	*/
4766	static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4767	{
4768	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0))
4769	{
4770	uint32_t uVal = 0;
4771	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
4772	AssertRCReturn(rc, rc);
4773	uint32_t uShadow = 0;
4774	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4775	AssertRCReturn(rc, rc);
4776
4777	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
4778	CPUMSetGuestCR0(pVCpu, uVal);
4779	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR0;
4780	}
4781	return VINF_SUCCESS;
4782	}
4783
4784
4785	/**
4786	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4787	*
4788	* @returns VBox status code.
4789	* @param pVCpu Pointer to the VMCPU.
4790	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4791	* out-of-sync. Make sure to update the required fields
4792	* before using them.
4793	*
4794	* @remarks No-long-jump zone!!!
4795	*/
4796	static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4797	{
4798	int rc = VINF_SUCCESS;
4799	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4))
4800	{
4801	uint32_t uVal = 0;
4802	uint32_t uShadow = 0;
4803	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
4804	AssertRCReturn(rc, rc);
4805	rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4806	AssertRCReturn(rc, rc);
4807
4808	uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
4809	CPUMSetGuestCR4(pVCpu, uVal);
4810	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR4;
4811	}
4812	return rc;
4813	}
4814
4815
4816	/**
4817	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4818	*
4819	* @returns VBox status code.
4820	* @param pVCpu Pointer to the VMCPU.
4821	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4822	* out-of-sync. Make sure to update the required fields
4823	* before using them.
4824	*
4825	* @remarks No-long-jump zone!!!
4826	*/
4827	static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4828	{
4829	int rc = VINF_SUCCESS;
4830	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP))
4831	{
4832	uint64_t u64Val = 0;
4833	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
4834	AssertRCReturn(rc, rc);
4835
4836	pMixedCtx->rip = u64Val;
4837	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RIP;
4838	}
4839	return rc;
4840	}
4841
4842
4843	/**
4844	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4845	*
4846	* @returns VBox status code.
4847	* @param pVCpu Pointer to the VMCPU.
4848	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4849	* out-of-sync. Make sure to update the required fields
4850	* before using them.
4851	*
4852	* @remarks No-long-jump zone!!!
4853	*/
4854	static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4855	{
4856	int rc = VINF_SUCCESS;
4857	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RSP))
4858	{
4859	uint64_t u64Val = 0;
4860	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
4861	AssertRCReturn(rc, rc);
4862
4863	pMixedCtx->rsp = u64Val;
4864	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RSP;
4865	}
4866	return rc;
4867	}
4868
4869
4870	/**
4871	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4872	*
4873	* @returns VBox status code.
4874	* @param pVCpu Pointer to the VMCPU.
4875	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4876	* out-of-sync. Make sure to update the required fields
4877	* before using them.
4878	*
4879	* @remarks No-long-jump zone!!!
4880	*/
4881	static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4882	{
4883	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS))
4884	{
4885	uint32_t uVal = 0;
4886	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
4887	AssertRCReturn(rc, rc);
4888
4889	pMixedCtx->eflags.u32 = uVal;
4890	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
4891	{
4892	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4893	Log(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
4894
4895	pMixedCtx->eflags.Bits.u1VM = 0;
4896	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4897	}
4898
4899	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_RFLAGS;
4900	}
4901	return VINF_SUCCESS;
4902	}
4903
4904
4905	/**
4906	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4907	* guest-CPU context.
4908	*/
4909	DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4910	{
4911	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4912	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4913	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4914	return rc;
4915	}
4916
4917
4918	/**
4919	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4920	* from the guest-state area in the VMCS.
4921	*
4922	* @param pVCpu Pointer to the VMCPU.
4923	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4924	* out-of-sync. Make sure to update the required fields
4925	* before using them.
4926	*
4927	* @remarks No-long-jump zone!!!
4928	*/
4929	static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4930	{
4931	uint32_t uIntrState = 0;
4932	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4933	AssertRC(rc);
4934
4935	if (!uIntrState)
4936	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4937	else
4938	{
4939	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4940	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4941	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4942	AssertRC(rc);
4943	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
4944	AssertRC(rc);
4945
4946	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4947	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4948	}
4949	}
4950
4951
4952	/**
4953	* Saves the guest's activity state.
4954	*
4955	* @returns VBox status code.
4956	* @param pVCpu Pointer to the VMCPU.
4957	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4958	* out-of-sync. Make sure to update the required fields
4959	* before using them.
4960	*
4961	* @remarks No-long-jump zone!!!
4962	*/
4963	static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4964	{
4965	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4966	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_ACTIVITY_STATE;
4967	return VINF_SUCCESS;
4968	}
4969
4970
4971	/**
4972	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4973	* the current VMCS into the guest-CPU context.
4974	*
4975	* @returns VBox status code.
4976	* @param pVCpu Pointer to the VMCPU.
4977	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4978	* out-of-sync. Make sure to update the required fields
4979	* before using them.
4980	*
4981	* @remarks No-long-jump zone!!!
4982	*/
4983	static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4984	{
4985	int rc = VINF_SUCCESS;
4986	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4987	{
4988	uint32_t u32Val = 0;
4989	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4990	pMixedCtx->SysEnter.cs = u32Val;
4991	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4992	}
4993
4994	uint64_t u64Val = 0;
4995	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4996	{
4997	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
4998	pMixedCtx->SysEnter.eip = u64Val;
4999	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5000	}
5001	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5002	{
5003	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
5004	pMixedCtx->SysEnter.esp = u64Val;
5005	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5006	}
5007	return rc;
5008	}
5009
5010
5011	/**
5012	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5013	* context.
5014	*
5015	* @returns VBox status code.
5016	* @param pVCpu Pointer to the VMCPU.
5017	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5018	* out-of-sync. Make sure to update the required fields
5019	* before using them.
5020	*
5021	* @remarks No-long-jump zone!!!
5022	*/
5023	static int hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5024	{
5025	int rc = VINF_SUCCESS;
5026	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_FS_BASE_MSR))
5027	{
5028	uint64_t u64Val = 0;
5029	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &u64Val); AssertRCReturn(rc, rc);
5030	pMixedCtx->fs.u64Base = u64Val;
5031	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_FS_BASE_MSR;
5032	}
5033	return rc;
5034	}
5035
5036
5037	/**
5038	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5039	* context.
5040	*
5041	* @returns VBox status code.
5042	* @param pVCpu Pointer to the VMCPU.
5043	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5044	* out-of-sync. Make sure to update the required fields
5045	* before using them.
5046	*
5047	* @remarks No-long-jump zone!!!
5048	*/
5049	static int hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5050	{
5051	int rc = VINF_SUCCESS;
5052	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GS_BASE_MSR))
5053	{
5054	uint64_t u64Val = 0;
5055	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &u64Val); AssertRCReturn(rc, rc);
5056	pMixedCtx->gs.u64Base = u64Val;
5057	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GS_BASE_MSR;
5058	}
5059	return rc;
5060	}
5061
5062
5063	/**
5064	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5065	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK, KERNEL-GS BASE
5066	* and TSC_AUX.
5067	*
5068	* @returns VBox status code.
5069	* @param pVCpu Pointer to the VMCPU.
5070	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5071	* out-of-sync. Make sure to update the required fields
5072	* before using them.
5073	*
5074	* @remarks No-long-jump zone!!!
5075	*/
5076	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5077	{
5078	if (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5079	return VINF_SUCCESS;
5080
5081	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
5082	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5083	{
5084	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5085	pMsr += i;
5086	switch (pMsr->u32IndexMSR)
5087	{
5088	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5089	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5090	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5091	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5092	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5093	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5094	default:
5095	{
5096	AssertFailed();
5097	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5098	}
5099	}
5100	}
5101	#endif
5102
5103	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5104	return VINF_SUCCESS;
5105	}
5106
5107
5108	/**
5109	* Saves the guest control registers from the current VMCS into the guest-CPU
5110	* context.
5111	*
5112	* @returns VBox status code.
5113	* @param pVCpu Pointer to the VMCPU.
5114	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5115	* out-of-sync. Make sure to update the required fields
5116	* before using them.
5117	*
5118	* @remarks No-long-jump zone!!!
5119	*/
5120	static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5121	{
5122	/* Guest CR0. Guest FPU. */
5123	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5124	AssertRCReturn(rc, rc);
5125
5126	/* Guest CR4. */
5127	rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5128	AssertRCReturn(rc, rc);
5129
5130	/* Guest CR2 - updated always during the world-switch or in #PF. */
5131	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5132	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR3))
5133	{
5134	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR0);
5135	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_CR4);
5136
5137	PVM pVM = pVCpu->CTX_SUFF(pVM);
5138	if ( pVM->hm.s.vmx.fUnrestrictedGuest
5139	\|\| ( pVM->hm.s.fNestedPaging
5140	&& CPUMIsGuestPagingEnabledEx(pMixedCtx)))
5141	{
5142	uint64_t u64Val = 0;
5143	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
5144	if (pMixedCtx->cr3 != u64Val)
5145	{
5146	CPUMSetGuestCR3(pVCpu, u64Val);
5147	if (VMMRZCallRing3IsEnabled(pVCpu))
5148	{
5149	PGMUpdateCR3(pVCpu, u64Val);
5150	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5151	}
5152	else
5153	{
5154	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5155	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5156	}
5157	}
5158
5159	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5160	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
5161	{
5162	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
5163	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
5164	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
5165	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
5166
5167	if (VMMRZCallRing3IsEnabled(pVCpu))
5168	{
5169	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5170	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5171	}
5172	else
5173	{
5174	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5175	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5176	}
5177	}
5178	}
5179
5180	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_CR3;
5181	}
5182
5183	/*
5184	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
5185	* -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
5186	* -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
5187	*
5188	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
5189	*/
5190	if (VMMRZCallRing3IsEnabled(pVCpu))
5191	{
5192	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5193	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
5194
5195	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5196	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5197
5198	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5199	Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5200	}
5201
5202	return rc;
5203	}
5204
5205
5206	/**
5207	* Reads a guest segment register from the current VMCS into the guest-CPU
5208	* context.
5209	*
5210	* @returns VBox status code.
5211	* @param pVCpu Pointer to the VMCPU.
5212	* @param idxSel Index of the selector in the VMCS.
5213	* @param idxLimit Index of the segment limit in the VMCS.
5214	* @param idxBase Index of the segment base in the VMCS.
5215	* @param idxAccess Index of the access rights of the segment in the VMCS.
5216	* @param pSelReg Pointer to the segment selector.
5217	*
5218	* @remarks No-long-jump zone!!!
5219	* @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
5220	* macro as that takes care of whether to read from the VMCS cache or
5221	* not.
5222	*/
5223	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5224	PCPUMSELREG pSelReg)
5225	{
5226	uint32_t u32Val = 0;
5227	int rc = VMXReadVmcs32(idxSel, &u32Val);
5228	AssertRCReturn(rc, rc);
5229	pSelReg->Sel = (uint16_t)u32Val;
5230	pSelReg->ValidSel = (uint16_t)u32Val;
5231	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5232
5233	rc = VMXReadVmcs32(idxLimit, &u32Val);
5234	AssertRCReturn(rc, rc);
5235	pSelReg->u32Limit = u32Val;
5236
5237	uint64_t u64Val = 0;
5238	rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
5239	AssertRCReturn(rc, rc);
5240	pSelReg->u64Base = u64Val;
5241
5242	rc = VMXReadVmcs32(idxAccess, &u32Val);
5243	AssertRCReturn(rc, rc);
5244	pSelReg->Attr.u = u32Val;
5245
5246	/*
5247	* If VT-x marks the segment as unusable, the rest of the attributes are undefined with certain exceptions (some bits in
5248	* CS, SS). Regardless, we have to clear the bits here and only retain the unusable bit because the unusable bit is specific
5249	* to VT-x, everyone else relies on the attribute being zero and have no clue what the unusable bit is.
5250	*
5251	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
5252	*/
5253	if (pSelReg->Attr.u & HMVMX_SEL_UNUSABLE)
5254	{
5255	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
5256	pSelReg->Attr.u = HMVMX_SEL_UNUSABLE;
5257	}
5258	return VINF_SUCCESS;
5259	}
5260
5261
5262	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5263	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5264	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5265	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5266	#else
5267	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5268	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5269	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5270	#endif
5271
5272
5273	/**
5274	* Saves the guest segment registers from the current VMCS into the guest-CPU
5275	* context.
5276	*
5277	* @returns VBox status code.
5278	* @param pVCpu Pointer to the VMCPU.
5279	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5280	* out-of-sync. Make sure to update the required fields
5281	* before using them.
5282	*
5283	* @remarks No-long-jump zone!!!
5284	*/
5285	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5286	{
5287	/* Guest segment registers. */
5288	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_SEGMENT_REGS))
5289	{
5290	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
5291	rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
5292	rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
5293	rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
5294	rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
5295	rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
5296	rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
5297
5298	/* Restore segment attributes for real-on-v86 mode hack. */
5299	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5300	{
5301	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5302	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5303	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5304	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5305	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5306	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5307	}
5308	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_SEGMENT_REGS;
5309	}
5310
5311	return VINF_SUCCESS;
5312	}
5313
5314
5315	/**
5316	* Saves the guest descriptor table registers and task register from the current
5317	* VMCS into the guest-CPU context.
5318	*
5319	* @returns VBox status code.
5320	* @param pVCpu Pointer to the VMCPU.
5321	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5322	* out-of-sync. Make sure to update the required fields
5323	* before using them.
5324	*
5325	* @remarks No-long-jump zone!!!
5326	*/
5327	static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5328	{
5329	int rc = VINF_SUCCESS;
5330
5331	/* Guest LDTR. */
5332	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_LDTR))
5333	{
5334	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5335	AssertRCReturn(rc, rc);
5336	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_LDTR;
5337	}
5338
5339	/* Guest GDTR. */
5340	uint64_t u64Val = 0;
5341	uint32_t u32Val = 0;
5342	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_GDTR))
5343	{
5344	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5345	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5346	pMixedCtx->gdtr.pGdt = u64Val;
5347	pMixedCtx->gdtr.cbGdt = u32Val;
5348	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_GDTR;
5349	}
5350
5351	/* Guest IDTR. */
5352	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_IDTR))
5353	{
5354	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
5355	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5356	pMixedCtx->idtr.pIdt = u64Val;
5357	pMixedCtx->idtr.cbIdt = u32Val;
5358	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_IDTR;
5359	}
5360
5361	/* Guest TR. */
5362	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_TR))
5363	{
5364	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5365	AssertRCReturn(rc, rc);
5366
5367	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5368	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5369	{
5370	rc = VMXLOCAL_READ_SEG(TR, tr);
5371	AssertRCReturn(rc, rc);
5372	}
5373	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_TR;
5374	}
5375	return rc;
5376	}
5377
5378	#undef VMXLOCAL_READ_SEG
5379
5380
5381	/**
5382	* Saves the guest debug registers from the current VMCS into the guest-CPU
5383	* context.
5384	*
5385	* @returns VBox status code.
5386	* @param pVCpu Pointer to the VMCPU.
5387	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5388	* out-of-sync. Make sure to update the required fields
5389	* before using them.
5390	*
5391	* @remarks No-long-jump zone!!!
5392	*/
5393	static int hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5394	{
5395	int rc = VINF_SUCCESS;
5396	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_DEBUG))
5397	{
5398	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
5399	uint32_t u32Val;
5400	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
5401	pMixedCtx->dr[7] = u32Val;
5402
5403	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_DEBUG;
5404	}
5405	return rc;
5406	}
5407
5408
5409	/**
5410	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5411	*
5412	* @returns VBox status code.
5413	* @param pVCpu Pointer to the VMCPU.
5414	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5415	* out-of-sync. Make sure to update the required fields
5416	* before using them.
5417	*
5418	* @remarks No-long-jump zone!!!
5419	*/
5420	static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5421	{
5422	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5423	pVCpu->hm.s.vmx.fUpdatedGuestState \|= HMVMX_UPDATED_GUEST_APIC_STATE;
5424	return VINF_SUCCESS;
5425	}
5426
5427
5428	/**
5429	* Saves the entire guest state from the currently active VMCS into the
5430	* guest-CPU context. This essentially VMREADs all guest-data.
5431	*
5432	* @returns VBox status code.
5433	* @param pVCpu Pointer to the VMCPU.
5434	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5435	* out-of-sync. Make sure to update the required fields
5436	* before using them.
5437	*/
5438	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5439	{
5440	Assert(pVCpu);
5441	Assert(pMixedCtx);
5442
5443	if (pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL)
5444	return VINF_SUCCESS;
5445
5446	/* Though we can longjmp to ring-3 due to log-flushes here and get recalled again on the ring-3 callback path,
5447	there is no real need to. */
5448	if (VMMRZCallRing3IsEnabled(pVCpu))
5449	VMMR0LogFlushDisable(pVCpu);
5450	else
5451	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5452	LogFunc(("\n"));
5453
5454	int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
5455	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5456
5457	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5458	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5459
5460	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5462
5463	rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
5464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5465
5466	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5467	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5468
5469	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5470	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5471
5472	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5473	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5474
5475	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5476	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5477
5478	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5479	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5480
5481	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5482	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5483
5484	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5485	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5486
5487	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL,
5488	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5489
5490	if (VMMRZCallRing3IsEnabled(pVCpu))
5491	VMMR0LogFlushEnable(pVCpu);
5492
5493	return rc;
5494	}
5495
5496
5497	/**
5498	* Check per-VM and per-VCPU force flag actions that require us to go back to
5499	* ring-3 for one reason or another.
5500	*
5501	* @returns VBox status code (information status code included).
5502	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5503	* ring-3.
5504	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5505	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5506	* interrupts)
5507	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5508	* all EMTs to be in ring-3.
5509	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5510	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5511	* to the EM loop.
5512	*
5513	* @param pVM Pointer to the VM.
5514	* @param pVCpu Pointer to the VMCPU.
5515	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5516	* out-of-sync. Make sure to update the required fields
5517	* before using them.
5518	*/
5519	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5520	{
5521	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5522
5523	int rc = VERR_INTERNAL_ERROR_5;
5524	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5525	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5526	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5527	{
5528	/* We need the control registers now, make sure the guest-CPU context is updated. */
5529	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5530	AssertRCReturn(rc, rc);
5531
5532	/* Pending HM CR3 sync. */
5533	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5534	{
5535	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5536	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5537	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
5538	}
5539
5540	/* Pending HM PAE PDPEs. */
5541	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5542	{
5543	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5544	AssertRC(rc);
5545	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
5546	}
5547
5548	/* Pending PGM C3 sync. */
5549	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5550	{
5551	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5552	if (rc != VINF_SUCCESS)
5553	{
5554	AssertRC(rc);
5555	Log(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5556	return rc;
5557	}
5558	}
5559
5560	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5561	/* -XXX- what was that about single stepping? */
5562	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5563	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5564	{
5565	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5566	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5567	Log(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5568	return rc;
5569	}
5570
5571	/* Pending VM request packets, such as hardware interrupts. */
5572	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5573	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5574	{
5575	Log(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5576	return VINF_EM_PENDING_REQUEST;
5577	}
5578
5579	/* Pending PGM pool flushes. */
5580	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5581	{
5582	Log(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5583	return VINF_PGM_POOL_FLUSH_PENDING;
5584	}
5585
5586	/* Pending DMA requests. */
5587	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5588	{
5589	Log(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5590	return VINF_EM_RAW_TO_R3;
5591	}
5592	}
5593
5594	/* Paranoia. */
5595	Assert(rc != VERR_EM_INTERPRETER);
5596	return VINF_SUCCESS;
5597	}
5598
5599
5600	/**
5601	* Converts any TRPM trap into a pending VMX event. This is typically used when
5602	* entering from ring-3 (not longjmp returns).
5603	*
5604	* @param pVCpu Pointer to the VMCPU.
5605	*/
5606	static void hmR0VmxTRPMTrapToPendingEvent(PVMCPU pVCpu)
5607	{
5608	Assert(TRPMHasTrap(pVCpu));
5609	Assert(!pVCpu->hm.s.Event.fPending);
5610
5611	uint8_t uVector;
5612	TRPMEVENT enmTrpmEvent;
5613	RTGCUINT uErrCode;
5614	RTGCUINTPTR GCPtrFaultAddress;
5615	uint8_t cbInstr;
5616
5617	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5618	AssertRC(rc);
5619
5620	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5621	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5622	if (enmTrpmEvent == TRPM_TRAP)
5623	{
5624	switch (uVector)
5625	{
5626	case X86_XCPT_BP:
5627	case X86_XCPT_OF:
5628	{
5629	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5630	break;
5631	}
5632
5633	case X86_XCPT_PF:
5634	case X86_XCPT_DF:
5635	case X86_XCPT_TS:
5636	case X86_XCPT_NP:
5637	case X86_XCPT_SS:
5638	case X86_XCPT_GP:
5639	case X86_XCPT_AC:
5640	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5641	/* no break! */
5642	default:
5643	{
5644	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5645	break;
5646	}
5647	}
5648	}
5649	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5650	{
5651	if (uVector != X86_XCPT_NMI)
5652	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5653	else
5654	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5655	}
5656	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5657	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5658	else
5659	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5660
5661	rc = TRPMResetTrap(pVCpu);
5662	AssertRC(rc);
5663	Log(("TRPM->HM event: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5664	u32IntrInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
5665	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, uErrCode, GCPtrFaultAddress);
5666	}
5667
5668
5669	/**
5670	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5671	* VT-x to execute any instruction.
5672	*
5673	* @param pvCpu Pointer to the VMCPU.
5674	*/
5675	static void hmR0VmxPendingEventToTRPMTrap(PVMCPU pVCpu)
5676	{
5677	Assert(pVCpu->hm.s.Event.fPending);
5678
5679	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5680	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5681	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5682	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5683
5684	/* If a trap was already pending, we did something wrong! */
5685	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5686
5687	TRPMEVENT enmTrapType;
5688	switch (uVectorType)
5689	{
5690	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5691	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5692	enmTrapType = TRPM_HARDWARE_INT;
5693	break;
5694	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5695	enmTrapType = TRPM_SOFTWARE_INT;
5696	break;
5697	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5698	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5699	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5700	enmTrapType = TRPM_TRAP;
5701	break;
5702	default:
5703	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5704	enmTrapType = TRPM_32BIT_HACK;
5705	break;
5706	}
5707
5708	Log(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5709
5710	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5711	AssertRC(rc);
5712
5713	if (fErrorCodeValid)
5714	TRPMSetErrorCode(pVCpu, uErrorCode);
5715	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5716	&& uVector == X86_XCPT_PF)
5717	{
5718	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5719	}
5720	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5721	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5722	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5723	{
5724	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5725	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5726	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5727	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5728	}
5729	pVCpu->hm.s.Event.fPending = false;
5730	}
5731
5732
5733	/**
5734	* Does the necessary state syncing before doing a longjmp to ring-3.
5735	*
5736	* @param pVM Pointer to the VM.
5737	* @param pVCpu Pointer to the VMCPU.
5738	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5739	* out-of-sync. Make sure to update the required fields
5740	* before using them.
5741	* @param rcExit The reason for exiting to ring-3. Can be
5742	* VINF_VMM_UNKNOWN_RING3_CALL.
5743	*
5744	* @remarks No-long-jmp zone!!!
5745	*/
5746	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5747	{
5748	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5749	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5750
5751	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5752	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == HMVMX_UPDATED_GUEST_ALL);
5753	AssertRC(rc);
5754
5755	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5756	if (CPUMIsGuestFPUStateActive(pVCpu))
5757	{
5758	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5759	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5760	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5761	}
5762
5763	/* Restore debug registers if necessary and resync on next R0 reentry. */
5764	if (CPUMIsGuestDebugStateActive(pVCpu))
5765	{
5766	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5767	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5768	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5769	}
5770	else if (CPUMIsHyperDebugStateActive(pVCpu))
5771	{
5772	CPUMR0LoadHostDebugState(pVM, pVCpu);
5773	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5774	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5775	}
5776
5777	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5778	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5779	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5780	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5781	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5782	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5783	}
5784
5785
5786	/**
5787	* An action requires us to go back to ring-3. This function does the necessary
5788	* steps before we can safely return to ring-3. This is not the same as longjmps
5789	* to ring-3, this is voluntary.
5790	*
5791	* @param pVM Pointer to the VM.
5792	* @param pVCpu Pointer to the VMCPU.
5793	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5794	* out-of-sync. Make sure to update the required fields
5795	* before using them.
5796	* @param rcExit The reason for exiting to ring-3. Can be
5797	* VINF_VMM_UNKNOWN_RING3_CALL.
5798	*/
5799	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5800	{
5801	Assert(pVM);
5802	Assert(pVCpu);
5803	Assert(pMixedCtx);
5804	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5805
5806	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5807	{
5808	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5809	return;
5810	}
5811	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5812	{
5813	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5814	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5815	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5816	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5817	return;
5818	}
5819
5820	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5821	VMMRZCallRing3Disable(pVCpu);
5822	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5823
5824	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5825	if (pVCpu->hm.s.Event.fPending)
5826	{
5827	hmR0VmxPendingEventToTRPMTrap(pVCpu);
5828	Assert(!pVCpu->hm.s.Event.fPending);
5829	}
5830
5831	/* Sync. the guest state. */
5832	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5833	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5834
5835	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5836	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5837	\| CPUM_CHANGED_LDTR
5838	\| CPUM_CHANGED_GDTR
5839	\| CPUM_CHANGED_IDTR
5840	\| CPUM_CHANGED_TR
5841	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5842
5843	/* On our way back from ring-3 the following needs to be done. */
5844	/** @todo This can change with preemption hooks. */
5845	if (rcExit == VINF_EM_RAW_INTERRUPT)
5846	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5847	else
5848	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5849
5850	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5851	VMMRZCallRing3Enable(pVCpu);
5852	}
5853
5854
5855	/**
5856	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5857	* longjump to ring-3 and possibly get preempted.
5858	*
5859	* @param pVCpu Pointer to the VMCPU.
5860	* @param enmOperation The operation causing the ring-3 longjump.
5861	* @param pvUser The user argument (pointer to the possibly
5862	* out-of-date guest-CPU context).
5863	*
5864	* @remarks Must never be called with @a enmOperation ==
5865	* VMMCALLRING3_VM_R0_ASSERTION.
5866	*/
5867	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5868	{
5869	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5870	Assert(pVCpu);
5871	Assert(pvUser);
5872	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5873	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5874
5875	VMMRZCallRing3Disable(pVCpu);
5876	Assert(VMMR0IsLogFlushDisabled(pVCpu));
5877	Log(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3\n"));
5878	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5879	VMMRZCallRing3Enable(pVCpu);
5880	}
5881
5882
5883	/**
5884	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5885	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5886	*
5887	* @returns VBox status code.
5888	* @param pVCpu Pointer to the VMCPU.
5889	*/
5890	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5891	{
5892	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5893	{
5894	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
5895	{
5896	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
5897	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5898	AssertRC(rc);
5899	}
5900	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5901	}
5902
5903
5904	/**
5905	* Injects any pending events into the guest if the guest is in a state to
5906	* receive them.
5907	*
5908	* @returns VBox status code (informational status codes included).
5909	* @param pVCpu Pointer to the VMCPU.
5910	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5911	* out-of-sync. Make sure to update the required fields
5912	* before using them.
5913	*/
5914	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5915	{
5916	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5917	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5918	bool fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5919	bool fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5920
5921	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS));
5922	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5923	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5924	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5925	Assert(!TRPMHasTrap(pVCpu));
5926
5927	int rc = VINF_SUCCESS;
5928	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5929	{
5930	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5931	bool fInject = true;
5932	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5933	{
5934	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5935	AssertRCReturn(rc, rc);
5936	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5937	if ( fBlockInt
5938	\|\| fBlockSti
5939	\|\| fBlockMovSS)
5940	{
5941	fInject = false;
5942	}
5943	}
5944	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5945	&& ( fBlockMovSS
5946	\|\| fBlockSti))
5947	{
5948	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5949	fInject = false;
5950	}
5951
5952	if (fInject)
5953	{
5954	Log(("Injecting pending event\n"));
5955	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5956	pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
5957	AssertRCReturn(rc, rc);
5958	pVCpu->hm.s.Event.fPending = false;
5959	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
5960	}
5961	else
5962	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5963	} /** @todo SMI. SMIs take priority over NMIs. */
5964	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5965	{
5966	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5967	if ( !fBlockMovSS
5968	&& !fBlockSti)
5969	{
5970	Log(("Injecting NMI\n"));
5971	uint32_t u32IntrInfo = X86_XCPT_NMI \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5972	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5973	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5974	0 /* GCPtrFaultAddress */, &uIntrState);
5975	AssertRCReturn(rc, rc);
5976	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5977	}
5978	else
5979	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5980	}
5981	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5982	{
5983	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5984	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5985	AssertRCReturn(rc, rc);
5986	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5987	if ( !fBlockInt
5988	&& !fBlockSti
5989	&& !fBlockMovSS)
5990	{
5991	uint8_t u8Interrupt;
5992	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5993	if (RT_SUCCESS(rc))
5994	{
5995	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5996	uint32_t u32IntrInfo = u8Interrupt \| VMX_EXIT_INTERRUPTION_INFO_VALID;
5997	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5998	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */,
5999	0 /* GCPtrFaultAddress */, &uIntrState);
6000	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6001	}
6002	else
6003	{
6004	/** @todo Does this actually happen? If not turn it into an assertion. */
6005	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
6006	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
6007	rc = VINF_SUCCESS;
6008	}
6009	}
6010	else
6011	hmR0VmxSetIntWindowExitVmcs(pVCpu);
6012	}
6013
6014	/*
6015	* Delivery pending debug exception if the guest is single-stepping. The interruptibility-state could have been changed by
6016	* hmR0VmxInjectEventVmcs() (e.g. real-on-v86 injecting software interrupts), re-evaluate it and set the BS bit.
6017	*/
6018	fBlockMovSS = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6019	fBlockSti = !!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
6020	int rc2 = VINF_SUCCESS;
6021	if ( fBlockSti
6022	\|\| fBlockMovSS)
6023	{
6024	if (!DBGFIsStepping(pVCpu))
6025	{
6026	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RFLAGS);
6027	if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
6028	{
6029	/*
6030	* The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD, VMX_EXIT_MTF
6031	* VMX_EXIT_APIC_WRITE, VMX_EXIT_VIRTUALIZED_EOI. See Intel spec. 27.3.4 "Saving Non-Register State".
6032	*/
6033	rc2 = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
6034	AssertRCReturn(rc, rc);
6035	}
6036	}
6037	else
6038	{
6039	/* We are single-stepping in the hypervisor debugger, clear interrupt inhibition as setting the BS bit would mean
6040	delivering a #DB to the guest upon VM-entry when it shouldn't be. */
6041	uIntrState = 0;
6042	}
6043	}
6044
6045	/*
6046	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
6047	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
6048	*/
6049	rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6050	AssertRC(rc2);
6051
6052	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6053	return rc;
6054	}
6055
6056
6057	/**
6058	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6059	*
6060	* @param pVCpu Pointer to the VMCPU.
6061	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6062	* out-of-sync. Make sure to update the required fields
6063	* before using them.
6064	*/
6065	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6066	{
6067	uint32_t u32IntrInfo = X86_XCPT_UD \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6068	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6069	}
6070
6071
6072	/**
6073	* Injects a double-fault (#DF) exception into the VM.
6074	*
6075	* @returns VBox status code (informational status code included).
6076	* @param pVCpu Pointer to the VMCPU.
6077	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6078	* out-of-sync. Make sure to update the required fields
6079	* before using them.
6080	*/
6081	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6082	{
6083	uint32_t u32IntrInfo = X86_XCPT_DF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6084	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6085	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6086	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */,
6087	puIntrState);
6088	}
6089
6090
6091	/**
6092	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6093	*
6094	* @param pVCpu Pointer to the VMCPU.
6095	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6096	* out-of-sync. Make sure to update the required fields
6097	* before using them.
6098	*/
6099	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6100	{
6101	uint32_t u32IntrInfo = X86_XCPT_DB \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6102	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6103	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6104	}
6105
6106
6107	/**
6108	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6109	*
6110	* @param pVCpu Pointer to the VMCPU.
6111	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6112	* out-of-sync. Make sure to update the required fields
6113	* before using them.
6114	* @param cbInstr The value of RIP that is to be pushed on the guest
6115	* stack.
6116	*/
6117	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6118	{
6119	uint32_t u32IntrInfo = X86_XCPT_OF \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6120	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6121	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6122	}
6123
6124
6125	/**
6126	* Injects a general-protection (#GP) fault into the VM.
6127	*
6128	* @returns VBox status code (informational status code included).
6129	* @param pVCpu Pointer to the VMCPU.
6130	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6131	* out-of-sync. Make sure to update the required fields
6132	* before using them.
6133	* @param u32ErrorCode The error code associated with the #GP.
6134	*/
6135	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6136	uint32_t *puIntrState)
6137	{
6138	uint32_t u32IntrInfo = X86_XCPT_GP \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6139	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6140	if (fErrorCodeValid)
6141	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6142	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, u32ErrorCode, 0 / GCPtrFaultAddress */,
6143	puIntrState);
6144	}
6145
6146
6147	/**
6148	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6149	*
6150	* @param pVCpu Pointer to the VMCPU.
6151	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6152	* out-of-sync. Make sure to update the required fields
6153	* before using them.
6154	* @param uVector The software interrupt vector number.
6155	* @param cbInstr The value of RIP that is to be pushed on the guest
6156	* stack.
6157	*/
6158	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6159	{
6160	uint32_t u32IntrInfo = uVector \| VMX_EXIT_INTERRUPTION_INFO_VALID;
6161	if ( uVector == X86_XCPT_BP
6162	\|\| uVector == X86_XCPT_OF)
6163	{
6164	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6165	}
6166	else
6167	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6168	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6169	}
6170
6171
6172	/**
6173	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6174	* stack.
6175	*
6176	* @returns VBox status code (information status code included).
6177	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6178	* @param pVM Pointer to the VM.
6179	* @param pMixedCtx Pointer to the guest-CPU context.
6180	* @param uValue The value to push to the guest stack.
6181	*/
6182	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6183	{
6184	/*
6185	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6186	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6187	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6188	*/
6189	if (pMixedCtx->sp == 1)
6190	return VINF_EM_RESET;
6191	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6192	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6193	AssertRCReturn(rc, rc);
6194	return rc;
6195	}
6196
6197
6198	/**
6199	* Injects an event into the guest upon VM-entry by updating the relevant fields
6200	* in the VM-entry area in the VMCS.
6201	*
6202	* @returns VBox status code (informational error codes included).
6203	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6204	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6205	*
6206	* @param pVCpu Pointer to the VMCPU.
6207	* @param pMixedCtx Pointer to the guest-CPU context. The data may
6208	* be out-of-sync. Make sure to update the required
6209	* fields before using them.
6210	* @param u64IntrInfo The VM-entry interruption-information field.
6211	* @param cbInstr The VM-entry instruction length in bytes (for
6212	* software interrupts, exceptions and privileged
6213	* software exceptions).
6214	* @param u32ErrCode The VM-entry exception error code.
6215	* @param GCPtrFaultAddress The page-fault address for #PF exceptions.
6216	* @param puIntrState Pointer to the current guest interruptibility-state.
6217	* This interruptibility-state will be updated if
6218	* necessary. This cannot not be NULL.
6219	*
6220	* @remarks No-long-jump zone!!!
6221	* @remarks Requires CR0!
6222	*/
6223	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6224	uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
6225	{
6226	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6227	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6228	Assert(puIntrState);
6229	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6230
6231	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6232	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6233
6234	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6235	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6236	\|\| !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6237
6238	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6239
6240	/* We require CR0 to check if the guest is in real-mode. */
6241	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6242	AssertRCReturn(rc, rc);
6243
6244	/*
6245	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6246	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6247	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6248	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6249	*/
6250	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6251	{
6252	PVM pVM = pVCpu->CTX_SUFF(pVM);
6253	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6254	{
6255	Assert(PDMVmmDevHeapIsEnabled(pVM));
6256	Assert(pVM->hm.s.vmx.pRealModeTSS);
6257
6258	/* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
6259	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6260	rc \|= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6261	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6262	AssertRCReturn(rc, rc);
6263	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_RIP);
6264
6265	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6266	const size_t cbIdtEntry = 4;
6267	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6268	{
6269	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6270	if (uVector == X86_XCPT_DF)
6271	return VINF_EM_RESET;
6272	else if (uVector == X86_XCPT_GP)
6273	{
6274	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6275	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6276	}
6277
6278	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6279	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6280	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6281	}
6282
6283	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
6284	uint16_t uGuestIp = pMixedCtx->ip;
6285	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6286	{
6287	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6288	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6289	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6290	}
6291	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6292	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6293
6294	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6295	uint16_t offIdtEntry = 0;
6296	RTSEL selIdtEntry = 0;
6297	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6298	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6299	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6300	AssertRCReturn(rc, rc);
6301
6302	/* Construct the stack frame for the interrupt/exception handler. */
6303	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6304	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6305	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6306	AssertRCReturn(rc, rc);
6307
6308	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6309	if (rc == VINF_SUCCESS)
6310	{
6311	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6312	pMixedCtx->rip = offIdtEntry;
6313	pMixedCtx->cs.Sel = selIdtEntry;
6314	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6315	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6316	&& uVector == X86_XCPT_PF)
6317	{
6318	pMixedCtx->cr2 = GCPtrFaultAddress;
6319	}
6320	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6321	\| HM_CHANGED_GUEST_RIP
6322	\| HM_CHANGED_GUEST_RFLAGS
6323	\| HM_CHANGED_GUEST_RSP;
6324
6325	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6326	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6327	{
6328	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6329	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6330	Log(("Clearing inhibition due to STI.\n"));
6331	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6332	}
6333	Log(("Injecting real-mode: u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6334	}
6335	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6336	return rc;
6337	}
6338	else
6339	{
6340	/*
6341	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6342	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6343	*/
6344	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6345	}
6346	}
6347
6348	/* Validate. */
6349	Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6350	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6351	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6352
6353	/* Inject. */
6354	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6355	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6356	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6357	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6358
6359	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
6360	&& uVector == X86_XCPT_PF)
6361	{
6362	pMixedCtx->cr2 = GCPtrFaultAddress;
6363	}
6364	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x cbInstr=%#x uCR2=%#RX64\n", u32IntrInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
6365
6366	AssertRCReturn(rc, rc);
6367	return rc;
6368	}
6369
6370
6371	/**
6372	* Enters the VT-x session.
6373	*
6374	* @returns VBox status code.
6375	* @param pVM Pointer to the VM.
6376	* @param pVCpu Pointer to the VMCPU.
6377	* @param pCpu Pointer to the CPU info struct.
6378	*/
6379	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6380	{
6381	AssertPtr(pVM);
6382	AssertPtr(pVCpu);
6383	Assert(pVM->hm.s.vmx.fSupported);
6384	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6385	NOREF(pCpu);
6386
6387	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6388
6389	/* Make sure we're in VMX root mode. */
6390	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6391	if (!(u32HostCR4 & X86_CR4_VMXE))
6392	{
6393	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6394	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6395	}
6396
6397	/* Load the active VMCS as the current one. */
6398	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6399	if (RT_FAILURE(rc))
6400	return rc;
6401
6402	/** @todo this will change with preemption hooks where can can VMRESUME as long
6403	* as we're no preempted. */
6404	pVCpu->hm.s.fResumeVM = false;
6405	return VINF_SUCCESS;
6406	}
6407
6408
6409	/**
6410	* Leaves the VT-x session.
6411	*
6412	* @returns VBox status code.
6413	* @param pVM Pointer to the VM.
6414	* @param pVCpu Pointer to the VMCPU.
6415	* @param pCtx Pointer to the guest-CPU context.
6416	*/
6417	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6418	{
6419	AssertPtr(pVCpu);
6420	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6421	NOREF(pVM);
6422	NOREF(pCtx);
6423
6424	/** @todo this will change with preemption hooks where we only VMCLEAR when
6425	* we are actually going to be preempted, not all the time like we
6426	* currently do. */
6427	/*
6428	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6429	* and mark the VMCS launch-state as "clear".
6430	*/
6431	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6432	return rc;
6433	}
6434
6435
6436	/**
6437	* Saves the host state in the VMCS host-state.
6438	* Sets up the VM-exit MSR-load area.
6439	*
6440	* The CPU state will be loaded from these fields on every successful VM-exit.
6441	*
6442	* @returns VBox status code.
6443	* @param pVM Pointer to the VM.
6444	* @param pVCpu Pointer to the VMCPU.
6445	*
6446	* @remarks No-long-jump zone!!!
6447	*/
6448	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6449	{
6450	AssertPtr(pVM);
6451	AssertPtr(pVCpu);
6452	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6453
6454	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6455
6456	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6457	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6458	return VINF_SUCCESS;
6459
6460	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6462
6463	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6465
6466	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6467	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6468
6469	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6470	return rc;
6471	}
6472
6473
6474	/**
6475	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6476	* loaded from these fields on every successful VM-entry.
6477	*
6478	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6479	* Sets up the VM-entry controls.
6480	* Sets up the appropriate VMX non-root function to execute guest code based on
6481	* the guest CPU mode.
6482	*
6483	* @returns VBox status code.
6484	* @param pVM Pointer to the VM.
6485	* @param pVCpu Pointer to the VMCPU.
6486	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6487	* out-of-sync. Make sure to update the required fields
6488	* before using them.
6489	*
6490	* @remarks No-long-jump zone!!!
6491	*/
6492	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6493	{
6494	AssertPtr(pVM);
6495	AssertPtr(pVCpu);
6496	AssertPtr(pMixedCtx);
6497	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6498
6499	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6500
6501	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6502
6503	/* Determine real-on-v86 mode. */
6504	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6505	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6506	&& CPUMIsGuestInRealModeEx(pMixedCtx))
6507	{
6508	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6509	}
6510
6511	/*
6512	* Load the guest-state into the VMCS.
6513	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
6514	* Ideally, assert that the cross-dependent bits are up to date at the point of using it.
6515	*/
6516	int rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
6517	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6518
6519	rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
6520	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6521
6522	rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
6523	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6524
6525	rc = hmR0VmxLoadGuestControlRegs(pVCpu, pMixedCtx);
6526	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6527
6528	/* Must be done after CR0 is loaded (strict builds require CR0 for segment register validation checks). */
6529	rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
6530	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6531
6532	rc = hmR0VmxLoadGuestDebugRegs(pVCpu, pMixedCtx);
6533	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6534
6535	rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
6536	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6537
6538	rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
6539	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6540
6541	/* Must be done after hmR0VmxLoadGuestDebugRegs() as it may update eflags.TF for debugging purposes. */
6542	rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
6543	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6544
6545	rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
6546	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6547
6548	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6549	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6550	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6551
6552	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6553	return rc;
6554	}
6555
6556
6557	/**
6558	* Does the preparations before executing guest code in VT-x.
6559	*
6560	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6561	* recompiler. We must be cautious what we do here regarding committing
6562	* guest-state information into the the VMCS assuming we assuredly execute the
6563	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6564	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6565	* that the recompiler can (and should) use them when it resumes guest
6566	* execution. Otherwise such operations must be done when we can no longer
6567	* exit to ring-3.
6568	*
6569	* @returns VBox status code (informational status codes included).
6570	* @retval VINF_SUCCESS if we can proceed with running the guest.
6571	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6572	* into the guest.
6573	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6574	*
6575	* @param pVM Pointer to the VM.
6576	* @param pVCpu Pointer to the VMCPU.
6577	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6578	* out-of-sync. Make sure to update the required fields
6579	* before using them.
6580	* @param pVmxTransient Pointer to the VMX transient structure.
6581	*
6582	* @remarks Called with preemption disabled.
6583	*/
6584	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6585	{
6586	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6587
6588	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6589	PGMRZDynMapFlushAutoSet(pVCpu);
6590	#endif
6591
6592	/* Check force flag actions that might require us to go back to ring-3. */
6593	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6594	if (rc != VINF_SUCCESS)
6595	return rc;
6596
6597	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6598	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6599	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6600	{
6601	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6602	RTGCPHYS GCPhysApicBase;
6603	GCPhysApicBase = pMixedCtx->msrApicBase;
6604	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6605
6606	/* Unalias any existing mapping. */
6607	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6608	AssertRCReturn(rc, rc);
6609
6610	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6611	Log(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6612	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6613	AssertRCReturn(rc, rc);
6614
6615	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6616	}
6617
6618	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6619	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6620	pVmxTransient->uEFlags = ASMIntDisableFlags();
6621	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6622	{
6623	ASMSetFlags(pVmxTransient->uEFlags);
6624	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6625	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6626	return VINF_EM_RAW_INTERRUPT;
6627	}
6628	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6629	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6630	#endif
6631
6632	/*
6633	* Evaluates and injects any pending events, toggling force-flags and updating the guest-interruptibility
6634	* state (interrupt shadow) in the VMCS. This -can- potentially be reworked to be done before disabling
6635	* interrupts and handle returning to ring-3 afterwards, but requires very careful state restoration.
6636	*/
6637	/** @todo Rework event evaluation and injection to be completely separate. */
6638	if (TRPMHasTrap(pVCpu))
6639	hmR0VmxTRPMTrapToPendingEvent(pVCpu);
6640
6641	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6642	AssertRCReturn(rc, rc);
6643	return rc;
6644	}
6645
6646
6647	/**
6648	* Prepares to run guest code in VT-x and we've committed to doing so. This
6649	* means there is no backing out to ring-3 or anywhere else at this
6650	* point.
6651	*
6652	* @param pVM Pointer to the VM.
6653	* @param pVCpu Pointer to the VMCPU.
6654	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6655	* out-of-sync. Make sure to update the required fields
6656	* before using them.
6657	* @param pVmxTransient Pointer to the VMX transient structure.
6658	*
6659	* @remarks Called with preemption disabled.
6660	* @remarks No-long-jump zone!!!
6661	*/
6662	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6663	{
6664	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6665	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6666
6667	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6668	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6669	pVmxTransient->uEFlags = ASMIntDisableFlags();
6670	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6671	#endif
6672
6673	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6674	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6675	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6676	#ifdef HMVMX_SYNC_FULL_GUEST_STATE
6677	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
6678	#endif
6679	int rc = VINF_SUCCESS;
6680	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6681	{
6682	rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
6683	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6684	}
6685	else if (pVCpu->hm.s.fContextUseFlags)
6686	{
6687	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6688	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6689	}
6690	AssertRC(rc);
6691	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6692
6693	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6694	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6695	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6696
6697	if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
6698	\|\| HMR0GetCurrentCpu()->idCpu != pVCpu->hm.s.idLastCpu)
6699	{
6700	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pMixedCtx);
6701	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6702	}
6703
6704	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6705	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6706	Assert(HMR0GetCurrentCpu()->idCpu == pVCpu->hm.s.idLastCpu);
6707
6708	/*
6709	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6710	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6711	*/
6712	if (pVM->hm.s.fTPRPatchingActive)
6713	{
6714	Assert(!CPUMIsGuestInLongMode(pVCpu));
6715
6716	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6717	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6718	AssertRC(rc);
6719
6720	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6721	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6722	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6723	see hmR0VmxLoadGuestApicState(). */
6724	}
6725
6726	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6727	/*
6728	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
6729	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
6730	*/
6731	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6732	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6733	{
6734	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
6735	uint64_t u64HostTscAux = 0;
6736	int rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
6737	AssertRC(rc2);
6738	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
6739	}
6740	#endif
6741
6742	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6743	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6744	to start executing. */
6745	}
6746
6747
6748	/**
6749	* Performs some essential restoration of state after running guest code in
6750	* VT-x.
6751	*
6752	* @param pVM Pointer to the VM.
6753	* @param pVCpu Pointer to the VMCPU.
6754	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6755	* out-of-sync. Make sure to update the required fields
6756	* before using them.
6757	* @param pVmxTransient Pointer to the VMX transient structure.
6758	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6759	*
6760	* @remarks Called with interrupts disabled.
6761	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6762	* unconditionally when it is safe to do so.
6763	*/
6764	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6765	{
6766	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6767	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6768
6769	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6770	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6771	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6772	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6773	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6774
6775	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
6776	{
6777	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
6778	/* Restore host's TSC_AUX. */
6779	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
6780	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
6781	#endif
6782	/** @todo Find a way to fix hardcoding a guestimate. */
6783	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6784	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6785	}
6786
6787	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6788	Assert(!(ASMGetFlags() & X86_EFL_IF));
6789	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
6790
6791	/* Restore the effects of TPR patching if any. */
6792	if (pVM->hm.s.fTPRPatchingActive)
6793	{
6794	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6795	AssertRC(rc);
6796	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6797	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6798	}
6799
6800	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6801	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6802
6803	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6804	uint32_t uExitReason;
6805	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6806	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6807	AssertRC(rc);
6808	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6809	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6810
6811	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6812	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6813
6814	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6815	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6816	{
6817	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6818	return;
6819	}
6820
6821	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6822	{
6823	/* Update the guest interruptibility-state from the VMCS. */
6824	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6825	#if defined(HMVMX_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_SAVE_FULL_GUEST_STATE)
6826	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6827	AssertRC(rc);
6828	#endif
6829	/*
6830	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6831	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
6832	* why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
6833	* cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
6834	*/
6835	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
6836	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6837	{
6838	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6839	AssertRC(rc);
6840	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6841	}
6842	}
6843	}
6844
6845
6846	/**
6847	* Runs the guest code using VT-x.
6848	*
6849	* @returns VBox status code.
6850	* @param pVM Pointer to the VM.
6851	* @param pVCpu Pointer to the VMCPU.
6852	* @param pCtx Pointer to the guest-CPU context.
6853	*
6854	* @remarks Called with preemption disabled.
6855	*/
6856	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6857	{
6858	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6859	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6860
6861	VMXTRANSIENT VmxTransient;
6862	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6863	int rc = VERR_INTERNAL_ERROR_5;
6864	uint32_t cLoops = 0;
6865
6866	for (;; cLoops++)
6867	{
6868	Assert(!HMR0SuspendPending());
6869	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6870	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6871	(unsigned)RTMpCpuId(), cLoops));
6872
6873	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6874	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6875	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6876	if (rc != VINF_SUCCESS)
6877	break;
6878
6879	/*
6880	* No longjmps to ring-3 from this point on!!!
6881	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6882	* This also disables flushing of the R0-logger instance (if any).
6883	*/
6884	VMMRZCallRing3Disable(pVCpu);
6885	VMMRZCallRing3RemoveNotification(pVCpu);
6886	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6887
6888	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6889	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6890
6891	/*
6892	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6893	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6894	*/
6895	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6896	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6897	{
6898	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6899	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6900	return rc;
6901	}
6902
6903	/* Handle the VM-exit. */
6904	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6905	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6906	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6907	HMVMX_START_EXIT_DISPATCH_PROF();
6908	#ifdef HMVMX_USE_FUNCTION_TABLE
6909	rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
6910	#else
6911	rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
6912	#endif
6913	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6914	if (rc != VINF_SUCCESS)
6915	break;
6916	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6917	{
6918	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6919	rc = VINF_EM_RAW_INTERRUPT;
6920	break;
6921	}
6922	}
6923
6924	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6925	if (rc == VERR_EM_INTERPRETER)
6926	rc = VINF_EM_RAW_EMULATE_INSTR;
6927	else if (rc == VINF_EM_RESET)
6928	rc = VINF_EM_TRIPLE_FAULT;
6929	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6930	return rc;
6931	}
6932
6933
6934	#ifndef HMVMX_USE_FUNCTION_TABLE
6935	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6936	{
6937	int rc;
6938	switch (rcReason)
6939	{
6940	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6941	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6942	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6943	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6944	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6945	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6946	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6947	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6948	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6949	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6950	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6951	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6952	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6953	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6954	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6955	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6956	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6957	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6958	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6959	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6960	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6961	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6962	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6963	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6964	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6965	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6966	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6967	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6968	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6969	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6970	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6971	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6972	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6973
6974	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6975	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6976	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6977	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6978	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6979	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6980	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6981	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6982	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6983
6984	case VMX_EXIT_VMCALL:
6985	case VMX_EXIT_VMCLEAR:
6986	case VMX_EXIT_VMLAUNCH:
6987	case VMX_EXIT_VMPTRLD:
6988	case VMX_EXIT_VMPTRST:
6989	case VMX_EXIT_VMREAD:
6990	case VMX_EXIT_VMRESUME:
6991	case VMX_EXIT_VMWRITE:
6992	case VMX_EXIT_VMXOFF:
6993	case VMX_EXIT_VMXON:
6994	case VMX_EXIT_INVEPT:
6995	case VMX_EXIT_INVVPID:
6996	case VMX_EXIT_VMFUNC:
6997	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6998	break;
6999	default:
7000	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
7001	break;
7002	}
7003	return rc;
7004	}
7005	#endif
7006
7007	#ifdef DEBUG
7008	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
7009	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
7010	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
7011	# define VMX_ASSERT_PREEMPT_CPUID() \
7012	do \
7013	{ \
7014	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
7015	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
7016	} while (0)
7017
7018	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
7019	do { \
7020	AssertPtr(pVCpu); \
7021	AssertPtr(pMixedCtx); \
7022	AssertPtr(pVmxTransient); \
7023	Assert(pVmxTransient->fVMEntryFailed == false); \
7024	Assert(ASMIntAreEnabled()); \
7025	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7026	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
7027	LogFunc(("vcpu[%u] vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n", \
7028	(unsigned)pVCpu->idCpu)); \
7029	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
7030	if (VMMR0IsLogFlushDisabled(pVCpu)) \
7031	VMX_ASSERT_PREEMPT_CPUID(); \
7032	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
7033	} while (0)
7034	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
7035	do { \
7036	LogFunc(("\n")); \
7037	} while(0)
7038	#else /* Release builds */
7039	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { HMVMX_STOP_EXIT_DISPATCH_PROF(); } while(0)
7040	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
7041	#endif
7042
7043
7044	/**
7045	* Advances the guest RIP after reading it from the VMCS.
7046	*
7047	* @returns VBox status code.
7048	* @param pVCpu Pointer to the VMCPU.
7049	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
7050	* out-of-sync. Make sure to update the required fields
7051	* before using them.
7052	* @param pVmxTransient Pointer to the VMX transient structure.
7053	*
7054	* @remarks No-long-jump zone!!!
7055	*/
7056	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7057	{
7058	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7059	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7060	AssertRCReturn(rc, rc);
7061
7062	pMixedCtx->rip += pVmxTransient->cbInstr;
7063	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7064	return rc;
7065	}
7066
7067
7068	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7069	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
7070	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
7071	/**
7072	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
7073	*/
7074	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7075	{
7076	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7077	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
7078	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
7079	Assert(ASMIntAreEnabled());
7080	return VINF_SUCCESS;
7081	#else
7082	return VINF_EM_RAW_INTERRUPT;
7083	#endif
7084	}
7085
7086
7087	/**
7088	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
7089	*/
7090	HMVMX_EXIT_DECL hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7091	{
7092	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7093	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
7094
7095	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
7096	AssertRCReturn(rc, rc);
7097
7098	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
7099	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
7100	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7101
7102	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7103	{
7104	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7105	return VINF_EM_RAW_INTERRUPT;
7106	}
7107
7108	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
7109	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
7110	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
7111	{
7112	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7113	return VINF_SUCCESS;
7114	}
7115	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
7116	{
7117	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7118	return rc;
7119	}
7120
7121	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7122	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7123	switch (uIntrType)
7124	{
7125	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7126	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7127	/* no break */
7128	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
7129	{
7130	switch (uVector)
7131	{
7132	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7133	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7134	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7135	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7136	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7137	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7138	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
7139	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7140	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7141	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7142	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7143	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7144	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7145	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7146	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7147	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7148	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7149	#endif
7150	default:
7151	{
7152	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7153	AssertRCReturn(rc, rc);
7154
7155	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7156	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7157	{
7158	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7159	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7160	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7161	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7162	AssertRCReturn(rc, rc);
7163	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7164	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7165	0 /* GCPtrFaultAddress */);
7166	AssertRCReturn(rc, rc);
7167	}
7168	else
7169	{
7170	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7171	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7172	}
7173	break;
7174	}
7175	}
7176	break;
7177	}
7178
7179	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7180	default:
7181	{
7182	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7183	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7184	break;
7185	}
7186	}
7187	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7188	return rc;
7189	}
7190
7191
7192	/**
7193	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7194	*/
7195	HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7196	{
7197	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7198
7199	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7200	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7201	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7202	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7203	AssertRCReturn(rc, rc);
7204
7205	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7206	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7207	return VINF_SUCCESS;
7208	}
7209
7210
7211	/**
7212	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7213	*/
7214	HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7215	{
7216	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7217	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7218	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7219	}
7220
7221
7222	/**
7223	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7224	*/
7225	HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7226	{
7227	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7228	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7229	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7230	}
7231
7232
7233	/**
7234	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7235	*/
7236	HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7237	{
7238	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7239	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7240	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7241	}
7242
7243
7244	/**
7245	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7246	*/
7247	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7248	{
7249	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7250	PVM pVM = pVCpu->CTX_SUFF(pVM);
7251	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7252	if (RT_LIKELY(rc == VINF_SUCCESS))
7253	{
7254	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7255	Assert(pVmxTransient->cbInstr == 2);
7256	}
7257	else
7258	{
7259	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7260	rc = VERR_EM_INTERPRETER;
7261	}
7262	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7263	return rc;
7264	}
7265
7266
7267	/**
7268	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7269	*/
7270	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7271	{
7272	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7273	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7274	AssertRCReturn(rc, rc);
7275
7276	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7277	return VINF_EM_RAW_EMULATE_INSTR;
7278
7279	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7280	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7281	}
7282
7283
7284	/**
7285	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7286	*/
7287	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7288	{
7289	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7290	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7291	AssertRCReturn(rc, rc);
7292
7293	PVM pVM = pVCpu->CTX_SUFF(pVM);
7294	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7295	if (RT_LIKELY(rc == VINF_SUCCESS))
7296	{
7297	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7298	Assert(pVmxTransient->cbInstr == 2);
7299	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7300	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7301	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7302	}
7303	else
7304	{
7305	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7306	rc = VERR_EM_INTERPRETER;
7307	}
7308	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7309	return rc;
7310	}
7311
7312
7313	/**
7314	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7315	*/
7316	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7317	{
7318	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7319	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7320	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7321	AssertRCReturn(rc, rc);
7322
7323	PVM pVM = pVCpu->CTX_SUFF(pVM);
7324	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7325	if (RT_LIKELY(rc == VINF_SUCCESS))
7326	{
7327	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7328	Assert(pVmxTransient->cbInstr == 3);
7329	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7330	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
7331	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7332	}
7333	else
7334	{
7335	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7336	rc = VERR_EM_INTERPRETER;
7337	}
7338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7339	return rc;
7340	}
7341
7342
7343	/**
7344	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7345	*/
7346	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7347	{
7348	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7349	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7350	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7351	AssertRCReturn(rc, rc);
7352
7353	PVM pVM = pVCpu->CTX_SUFF(pVM);
7354	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7355	if (RT_LIKELY(rc == VINF_SUCCESS))
7356	{
7357	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7358	Assert(pVmxTransient->cbInstr == 2);
7359	}
7360	else
7361	{
7362	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7363	rc = VERR_EM_INTERPRETER;
7364	}
7365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7366	return rc;
7367	}
7368
7369
7370	/**
7371	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7372	*/
7373	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7374	{
7375	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7376	PVM pVM = pVCpu->CTX_SUFF(pVM);
7377	Assert(!pVM->hm.s.fNestedPaging);
7378
7379	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7380	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7381	AssertRCReturn(rc, rc);
7382
7383	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7384	rc = VBOXSTRICTRC_VAL(rc2);
7385	if (RT_LIKELY(rc == VINF_SUCCESS))
7386	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7387	else
7388	{
7389	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
7390	pVmxTransient->uExitQualification, rc));
7391	}
7392	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7393	return rc;
7394	}
7395
7396
7397	/**
7398	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7399	*/
7400	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7401	{
7402	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7403	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7404	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7405	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7406	AssertRCReturn(rc, rc);
7407
7408	PVM pVM = pVCpu->CTX_SUFF(pVM);
7409	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7410	if (RT_LIKELY(rc == VINF_SUCCESS))
7411	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7412	else
7413	{
7414	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7415	rc = VERR_EM_INTERPRETER;
7416	}
7417	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7418	return rc;
7419	}
7420
7421
7422	/**
7423	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7424	*/
7425	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7426	{
7427	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7428	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7429	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7430	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7431	AssertRCReturn(rc, rc);
7432
7433	PVM pVM = pVCpu->CTX_SUFF(pVM);
7434	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7435	rc = VBOXSTRICTRC_VAL(rc2);
7436	if (RT_LIKELY( rc == VINF_SUCCESS
7437	\|\| rc == VINF_EM_HALT))
7438	{
7439	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7440	AssertRCReturn(rc3, rc3);
7441
7442	if ( rc == VINF_EM_HALT
7443	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7444	{
7445	rc = VINF_SUCCESS;
7446	}
7447	}
7448	else
7449	{
7450	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7451	rc = VERR_EM_INTERPRETER;
7452	}
7453	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7454	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7456	return rc;
7457	}
7458
7459
7460	/**
7461	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7462	*/
7463	HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7464	{
7465	/*
7466	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7467	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7468	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7469	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7470	*/
7471	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7472	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7473	}
7474
7475
7476	/**
7477	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7478	*/
7479	HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7480	{
7481	/*
7482	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7483	* root operation. If we get there there is something funny going on.
7484	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7485	*/
7486	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7487	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7488	}
7489
7490
7491	/**
7492	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7493	*/
7494	HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7495	{
7496	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7497	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7498	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7499	}
7500
7501
7502	/**
7503	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7504	*/
7505	HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7506	{
7507	/*
7508	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7509	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7510	* See Intel spec. 25.3 "Other Causes of VM-exits".
7511	*/
7512	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7513	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7514	}
7515
7516
7517	/**
7518	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7519	* VM-exit.
7520	*/
7521	HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7522	{
7523	/*
7524	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7525	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7526	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7527	*/
7528	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7529	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7530	}
7531
7532
7533	/**
7534	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7535	* VM-exit.
7536	*/
7537	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7538	{
7539	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7540	return VINF_EM_RESET;
7541	}
7542
7543
7544	/**
7545	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7546	*/
7547	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7548	{
7549	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7550	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
7551	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7552	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7553	AssertRCReturn(rc, rc);
7554
7555	pMixedCtx->rip++;
7556	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7557	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7558	rc = VINF_SUCCESS;
7559	else
7560	rc = VINF_EM_HALT;
7561
7562	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7563	return rc;
7564	}
7565
7566
7567	/**
7568	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7569	*/
7570	HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7571	{
7572	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7573	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7574	return VINF_SUCCESS;
7575	}
7576
7577
7578	/**
7579	* VM-exit handler for expiry of the VMX preemption timer.
7580	*/
7581	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7582	{
7583	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7584
7585	/* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
7586	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7587
7588	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7589	PVM pVM = pVCpu->CTX_SUFF(pVM);
7590	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7592	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7593	}
7594
7595
7596	/**
7597	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7598	*/
7599	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7600	{
7601	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7602
7603	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7604	/** @todo check if XSETBV is supported by the recompiler. */
7605	return VERR_EM_INTERPRETER;
7606	}
7607
7608
7609	/**
7610	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7611	*/
7612	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7613	{
7614	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7615
7616	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7617	/** @todo implement EMInterpretInvpcid() */
7618	return VERR_EM_INTERPRETER;
7619	}
7620
7621
7622	/**
7623	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7624	* Error VM-exit.
7625	*/
7626	HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7627	{
7628	uint32_t uIntrState;
7629	HMVMXHCUINTREG uHCReg;
7630	uint64_t u64Val;
7631	uint32_t u32Val;
7632
7633	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7634	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7635	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7636	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7637	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7638	AssertRCReturn(rc, rc);
7639
7640	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7641	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7642	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7643	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7644
7645	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
7646	Log(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
7647	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7648	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7649	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7650	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7651	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7652	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7653	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7654	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7655	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7656	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7657
7658	PVM pVM = pVCpu->CTX_SUFF(pVM);
7659	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7660
7661	return VERR_VMX_INVALID_GUEST_STATE;
7662	}
7663
7664
7665	/**
7666	* VM-exit handler for VM-entry failure due to an MSR-load
7667	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7668	*/
7669	HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7670	{
7671	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7672	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7673	}
7674
7675
7676	/**
7677	* VM-exit handler for VM-entry failure due to a machine-check event
7678	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7679	*/
7680	HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7681	{
7682	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7683	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7684	}
7685
7686
7687	/**
7688	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7689	* theory.
7690	*/
7691	HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7692	{
7693	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7694	return VERR_VMX_UNDEFINED_EXIT_CODE;
7695	}
7696
7697
7698	/**
7699	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7700	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7701	* Conditional VM-exit.
7702	*/
7703	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7704	{
7705	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7706
7707	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7709	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7710	return VERR_EM_INTERPRETER;
7711	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7712	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7713	}
7714
7715
7716	/**
7717	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7718	*/
7719	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7720	{
7721	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7722
7723	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7724	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7725	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7726	return VERR_EM_INTERPRETER;
7727	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7728	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7729	}
7730
7731
7732	/**
7733	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7734	*/
7735	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7736	{
7737	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7738
7739	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7740	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7741	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7742	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7743	AssertRCReturn(rc, rc);
7744
7745	PVM pVM = pVCpu->CTX_SUFF(pVM);
7746	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7747	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7748	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7749	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7750
7751	if (RT_LIKELY(rc == VINF_SUCCESS))
7752	{
7753	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7754	Assert(pVmxTransient->cbInstr == 2);
7755	}
7756	return rc;
7757	}
7758
7759
7760	/**
7761	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7762	*/
7763	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7764	{
7765	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7766	PVM pVM = pVCpu->CTX_SUFF(pVM);
7767	int rc = VINF_SUCCESS;
7768
7769	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7770	if ( pVM->hm.s.fTPRPatchingActive
7771	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7772	{
7773	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7774	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7775	{
7776	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7777	AssertRC(rc);
7778	}
7779
7780	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7781	Assert(pVmxTransient->cbInstr == 2);
7782	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7783	return VINF_SUCCESS;
7784	}
7785
7786	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7787	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7788	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7789	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7790	AssertRCReturn(rc, rc);
7791	Log(("ecx=%#RX32\n", pMixedCtx->ecx));
7792
7793	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7794	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7795	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7796
7797	if (RT_LIKELY(rc == VINF_SUCCESS))
7798	{
7799	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7800
7801	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7802	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7803	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7804	{
7805	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState & HMVMX_UPDATED_GUEST_APIC_STATE);
7806	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7807	}
7808	else if (pMixedCtx->ecx == MSR_K6_EFER) /* EFER is the only MSR we auto-load but don't allow write-passthrough. */
7809	{
7810	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7811	AssertRCReturn(rc, rc);
7812	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS;
7813	}
7814	else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
7815	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7816
7817	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7818	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
7819	{
7820	switch (pMixedCtx->ecx)
7821	{
7822	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7823	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7824	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7825	case MSR_K8_FS_BASE: /* no break */
7826	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS; break;
7827	case MSR_K8_KERNEL_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_AUTO_MSRS; break;
7828	}
7829	}
7830	#ifdef VBOX_STRICT
7831	else
7832	{
7833	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7834	switch (pMixedCtx->ecx)
7835	{
7836	case MSR_IA32_SYSENTER_CS:
7837	case MSR_IA32_SYSENTER_EIP:
7838	case MSR_IA32_SYSENTER_ESP:
7839	case MSR_K8_FS_BASE:
7840	case MSR_K8_GS_BASE:
7841	{
7842	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
7843	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7844	}
7845
7846	case MSR_K8_LSTAR:
7847	case MSR_K6_STAR:
7848	case MSR_K8_SF_MASK:
7849	case MSR_K8_TSC_AUX:
7850	case MSR_K8_KERNEL_GS_BASE:
7851	{
7852	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
7853	pMixedCtx->ecx));
7854	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7855	}
7856	}
7857	}
7858	#endif /* VBOX_STRICT */
7859	}
7860	return rc;
7861	}
7862
7863
7864	/**
7865	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7866	*/
7867	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7868	{
7869	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7870
7871	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
7872	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7873	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
7874	return VERR_EM_INTERPRETER;
7875	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7876	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7877	}
7878
7879
7880	/**
7881	* VM-exit handler for when the TPR value is lowered below the specified
7882	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7883	*/
7884	HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7885	{
7886	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7887	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
7888
7889	/*
7890	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7891	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7892	* resume guest execution.
7893	*/
7894	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7895	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7896	return VINF_SUCCESS;
7897	}
7898
7899
7900	/**
7901	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7902	* VM-exit.
7903	*
7904	* @retval VINF_SUCCESS when guest execution can continue.
7905	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7906	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7907	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7908	* recompiler.
7909	*/
7910	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7911	{
7912	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7913	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7914	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7915	AssertRCReturn(rc, rc);
7916
7917	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7918	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7919	PVM pVM = pVCpu->CTX_SUFF(pVM);
7920	switch (uAccessType)
7921	{
7922	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7923	{
7924	#if 0
7925	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7926	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7927	#else
7928	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7929	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7930	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7931	#endif
7932	AssertRCReturn(rc, rc);
7933
7934	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7935	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7936	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7937	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7938
7939	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7940	{
7941	case 0: /* CR0 */
7942	Log(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
7943	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7944	break;
7945	case 2: /* C2 **/
7946	/* Nothing to do here, CR2 it's not part of the VMCS. */
7947	break;
7948	case 3: /* CR3 */
7949	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7950	Log(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
7951	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7952	break;
7953	case 4: /* CR4 */
7954	Log(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
7955	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7956	break;
7957	case 8: /* CR8 */
7958	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7959	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7960	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7961	break;
7962	default:
7963	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7964	break;
7965	}
7966
7967	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7968	break;
7969	}
7970
7971	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7972	{
7973	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7974	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7975	AssertRCReturn(rc, rc);
7976	Assert( !pVM->hm.s.fNestedPaging
7977	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7978	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7979
7980	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7981	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7982	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
7983
7984	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7985	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7986	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7987	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7988	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7989	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7990	break;
7991	}
7992
7993	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7994	{
7995	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7996	AssertRCReturn(rc, rc);
7997	rc = EMInterpretCLTS(pVM, pVCpu);
7998	AssertRCReturn(rc, rc);
7999	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8000	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
8001	Log(("CRX CLTS write rc=%d\n", rc));
8002	break;
8003	}
8004
8005	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
8006	{
8007	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8008	AssertRCReturn(rc, rc);
8009	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
8010	if (RT_LIKELY(rc == VINF_SUCCESS))
8011	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8012	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
8013	Log(("CRX LMSW write rc=%d\n", rc));
8014	break;
8015	}
8016
8017	default:
8018	{
8019	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
8020	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
8021	}
8022	}
8023
8024	/* Validate possible error codes. */
8025	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
8026	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
8027	if (RT_SUCCESS(rc))
8028	{
8029	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8030	AssertRCReturn(rc2, rc2);
8031	}
8032
8033	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
8034	return rc;
8035	}
8036
8037
8038	/**
8039	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
8040	* VM-exit.
8041	*/
8042	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8043	{
8044	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8045	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
8046
8047	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8048	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8049	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8050	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
8051	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
8052	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
8053	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
8054	AssertRCReturn(rc, rc);
8055	Log(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8056
8057	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
8058	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
8059	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
8060	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
8061	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
8062	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
8063	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
8064
8065	/* I/O operation lookup arrays. */
8066	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
8067	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
8068
8069	const uint32_t cbSize = s_aIOSize[uIOWidth];
8070	const uint32_t cbInstr = pVmxTransient->cbInstr;
8071	PVM pVM = pVCpu->CTX_SUFF(pVM);
8072	if (fIOString)
8073	{
8074	/* INS/OUTS - I/O String instruction. */
8075	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8076	/** @todo for now manually disassemble later optimize by getting the fields from
8077	* the VMCS. */
8078	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
8079	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
8080	* segment prefix info. */
8081	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
8082	if (RT_SUCCESS(rc))
8083	{
8084	if (fIOWrite)
8085	{
8086	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8087	(DISCPUMODE)pDis->uAddrMode, cbSize);
8088	rc = VBOXSTRICTRC_VAL(rc2);
8089	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
8090	}
8091	else
8092	{
8093	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
8094	(DISCPUMODE)pDis->uAddrMode, cbSize);
8095	rc = VBOXSTRICTRC_VAL(rc2);
8096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
8097	}
8098	}
8099	else
8100	{
8101	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
8102	rc = VINF_EM_RAW_EMULATE_INSTR;
8103	}
8104	}
8105	else
8106	{
8107	/* IN/OUT - I/O instruction. */
8108	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
8109	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
8110	if (fIOWrite)
8111	{
8112	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
8113	rc = VBOXSTRICTRC_VAL(rc2);
8114	if (rc == VINF_IOM_R3_IOPORT_WRITE)
8115	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8116	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
8117	}
8118	else
8119	{
8120	uint32_t u32Result = 0;
8121	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
8122	rc = VBOXSTRICTRC_VAL(rc2);
8123	if (IOM_SUCCESS(rc))
8124	{
8125	/* Save result of I/O IN instr. in AL/AX/EAX. */
8126	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
8127	}
8128	else if (rc == VINF_IOM_R3_IOPORT_READ)
8129	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
8130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
8131	}
8132	}
8133
8134	if (IOM_SUCCESS(rc))
8135	{
8136	pMixedCtx->rip += cbInstr;
8137	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8138	if (RT_LIKELY(rc == VINF_SUCCESS))
8139	{
8140	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
8141	AssertRCReturn(rc, rc);
8142
8143	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8144	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8145	{
8146	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8147	for (unsigned i = 0; i < 4; i++)
8148	{
8149	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8150	if ( ( uIOPort >= pMixedCtx->dr[i]
8151	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8152	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8153	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8154	{
8155	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8156	uint64_t uDR6 = ASMGetDR6();
8157
8158	/* Clear all breakpoint status flags and set the one we just hit. */
8159	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8160	uDR6 \|= (uint64_t)RT_BIT(i);
8161
8162	/*
8163	* Note: AMD64 Architecture Programmer's Manual 13.1:
8164	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8165	* be cleared by software after the contents have been read.
8166	*/
8167	ASMSetDR6(uDR6);
8168
8169	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8170	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8171
8172	/* Paranoia. */
8173	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* Upper 32 bits MBZ. */
8174	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8175	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8176
8177	/* Resync DR7 */
8178	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8179	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8180
8181	/* Set #DB to be injected into the VM and continue guest execution. */
8182	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8183	break;
8184	}
8185	}
8186	}
8187	}
8188	}
8189
8190	#ifdef DEBUG
8191	if (rc == VINF_IOM_R3_IOPORT_READ)
8192	Assert(!fIOWrite);
8193	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8194	Assert(fIOWrite);
8195	else
8196	{
8197	AssertMsg( RT_FAILURE(rc)
8198	\|\| rc == VINF_SUCCESS
8199	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8200	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8201	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8202	}
8203	#endif
8204
8205	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8206	return rc;
8207	}
8208
8209
8210	/**
8211	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8212	* VM-exit.
8213	*/
8214	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8215	{
8216	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8217
8218	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8219	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8220	AssertRCReturn(rc, rc);
8221	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8222	{
8223	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8224	AssertRCReturn(rc, rc);
8225	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8226	{
8227	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8228	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8229	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8230	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8231	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8232	{
8233	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
8234	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
8235
8236	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8237	Assert(!pVCpu->hm.s.Event.fPending);
8238	pVCpu->hm.s.Event.fPending = true;
8239	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8240	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8241	AssertRCReturn(rc, rc);
8242	if (fErrorCodeValid)
8243	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8244	else
8245	pVCpu->hm.s.Event.u32ErrCode = 0;
8246	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
8247	&& uVector == X86_XCPT_PF)
8248	{
8249	pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
8250	}
8251	Log(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
8252	}
8253	}
8254	}
8255	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8256	* emulation. */
8257	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8258	return VERR_EM_INTERPRETER;
8259	}
8260
8261
8262	/**
8263	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8264	*/
8265	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8266	{
8267	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8268	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
8269	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
8270	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8271	AssertRCReturn(rc, rc);
8272	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8273	return VINF_EM_DBG_STOP;
8274	}
8275
8276
8277	/**
8278	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8279	*/
8280	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8281	{
8282	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8283
8284	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8285	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8286	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8287	return VINF_SUCCESS;
8288	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8289	return rc;
8290
8291	#if 0
8292	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8293	* just sync the whole thing. */
8294	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8295	#else
8296	/* Aggressive state sync. for now. */
8297	rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8298	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8299	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8300	#endif
8301	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8302	AssertRCReturn(rc, rc);
8303
8304	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8305	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8306	switch (uAccessType)
8307	{
8308	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8309	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8310	{
8311	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8312	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8313	{
8314	AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
8315	}
8316
8317	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8318	GCPhys &= PAGE_BASE_GC_MASK;
8319	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8320	PVM pVM = pVCpu->CTX_SUFF(pVM);
8321	Log(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
8322	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
8323
8324	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8325	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8326	CPUMCTX2CORE(pMixedCtx), GCPhys);
8327	rc = VBOXSTRICTRC_VAL(rc2);
8328	Log(("ApicAccess rc=%d\n", rc));
8329	if ( rc == VINF_SUCCESS
8330	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8331	\|\| rc == VERR_PAGE_NOT_PRESENT)
8332	{
8333	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8334	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8335	rc = VINF_SUCCESS;
8336	}
8337	break;
8338	}
8339
8340	default:
8341	Log(("ApicAccess uAccessType=%#x\n", uAccessType));
8342	rc = VINF_EM_RAW_EMULATE_INSTR;
8343	break;
8344	}
8345
8346	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8347	return rc;
8348	}
8349
8350
8351	/**
8352	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8353	* VM-exit.
8354	*/
8355	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8356	{
8357	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8358
8359	/* We should -not- get this VM-exit if the guest is debugging. */
8360	if (CPUMIsGuestDebugStateActive(pVCpu))
8361	{
8362	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8363	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8364	}
8365
8366	int rc = VERR_INTERNAL_ERROR_5;
8367	if ( !DBGFIsStepping(pVCpu)
8368	&& !CPUMIsHyperDebugStateActive(pVCpu))
8369	{
8370	/* Don't intercept MOV DRx. */
8371	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
8372	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8373	AssertRCReturn(rc, rc);
8374
8375	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8376	PVM pVM = pVCpu->CTX_SUFF(pVM);
8377	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8378	AssertRC(rc);
8379	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8380
8381	#ifdef VBOX_WITH_STATISTICS
8382	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8383	AssertRCReturn(rc, rc);
8384	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8385	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8386	else
8387	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8388	#endif
8389	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8390	return VINF_SUCCESS;
8391	}
8392
8393	/*
8394	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8395	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8396	*/
8397	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8398	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8399	AssertRCReturn(rc, rc);
8400
8401	PVM pVM = pVCpu->CTX_SUFF(pVM);
8402	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8403	{
8404	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8405	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8406	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8407	if (RT_SUCCESS(rc))
8408	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8409	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8410	}
8411	else
8412	{
8413	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8414	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8415	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8416	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8417	}
8418
8419	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8420	if (RT_SUCCESS(rc))
8421	{
8422	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8423	AssertRCReturn(rc2, rc2);
8424	}
8425	return rc;
8426	}
8427
8428
8429	/**
8430	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8431	* Conditional VM-exit.
8432	*/
8433	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8434	{
8435	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8436	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8437
8438	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8439	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8440	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8441	return VINF_SUCCESS;
8442	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8443	return rc;
8444
8445	RTGCPHYS GCPhys = 0;
8446	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8447
8448	#if 0
8449	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8450	#else
8451	/* Aggressive state sync. for now. */
8452	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8453	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8454	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8455	#endif
8456	AssertRCReturn(rc, rc);
8457
8458	/*
8459	* If we succeed, resume guest execution.
8460	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8461	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8462	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8463	* weird case. See @bugref{6043}.
8464	*/
8465	PVM pVM = pVCpu->CTX_SUFF(pVM);
8466	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8467	rc = VBOXSTRICTRC_VAL(rc2);
8468	Log(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8469	if ( rc == VINF_SUCCESS
8470	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8471	\|\| rc == VERR_PAGE_NOT_PRESENT)
8472	{
8473	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8474	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8475	return VINF_SUCCESS;
8476	}
8477	return rc;
8478	}
8479
8480
8481	/**
8482	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8483	* VM-exit.
8484	*/
8485	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8486	{
8487	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8488	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8489
8490	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8491	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8492	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8493	return VINF_SUCCESS;
8494	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8495	return rc;
8496
8497	RTGCPHYS GCPhys = 0;
8498	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8499	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8500	#if 0
8501	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8502	#else
8503	/* Aggressive state sync. for now. */
8504	rc \|= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8505	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8506	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8507	#endif
8508	AssertRCReturn(rc, rc);
8509
8510	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8511	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
8512
8513	RTGCUINT uErrorCode = 0;
8514	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8515	uErrorCode \|= X86_TRAP_PF_ID;
8516	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8517	uErrorCode \|= X86_TRAP_PF_RW;
8518	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8519	uErrorCode \|= X86_TRAP_PF_P;
8520
8521	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8522
8523	Log(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
8524	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8525
8526	/* Handle the pagefault trap for the nested shadow table. */
8527	PVM pVM = pVCpu->CTX_SUFF(pVM);
8528	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8529	TRPMResetTrap(pVCpu);
8530
8531	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8532	if ( rc == VINF_SUCCESS
8533	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8534	\|\| rc == VERR_PAGE_NOT_PRESENT)
8535	{
8536	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8538	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8539	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8540	return VINF_SUCCESS;
8541	}
8542
8543	Log(("EPT return to ring-3 rc=%d\n"));
8544	return rc;
8545	}
8546
8547
8548	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8549	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8550	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8551	/**
8552	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8553	*/
8554	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8555	{
8556	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8557	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8558
8559	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8560	AssertRCReturn(rc, rc);
8561
8562	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8563	{
8564	/* Old-style FPU error reporting needs some extra work. */
8565	/** @todo don't fall back to the recompiler, but do it manually. */
8566	return VERR_EM_INTERPRETER;
8567	}
8568	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8569	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8570	return rc;
8571	}
8572
8573
8574	/**
8575	* VM-exit exception handler for #BP (Breakpoint exception).
8576	*/
8577	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8578	{
8579	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8580	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8581
8582	/** @todo Try optimize this by not saving the entire guest state unless
8583	* really needed. */
8584	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8585	AssertRCReturn(rc, rc);
8586
8587	PVM pVM = pVCpu->CTX_SUFF(pVM);
8588	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8589	if (rc == VINF_EM_RAW_GUEST_TRAP)
8590	{
8591	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8592	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8593	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8594	AssertRCReturn(rc, rc);
8595
8596	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8597	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8598	}
8599
8600	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8601	return rc;
8602	}
8603
8604
8605	/**
8606	* VM-exit exception handler for #DB (Debug exception).
8607	*/
8608	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8609	{
8610	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8611	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8612
8613	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8614	AssertRCReturn(rc, rc);
8615
8616	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8617	uint64_t uDR6 = X86_DR6_INIT_VAL;
8618	uDR6 \|= (pVmxTransient->uExitQualification
8619	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8620	PVM pVM = pVCpu->CTX_SUFF(pVM);
8621	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8622	if (rc == VINF_EM_RAW_GUEST_TRAP)
8623	{
8624	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8625	pMixedCtx->dr[6] = uDR6;
8626
8627	if (CPUMIsGuestDebugStateActive(pVCpu))
8628	ASMSetDR6(pMixedCtx->dr[6]);
8629
8630	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
8631
8632	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8633	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8634
8635	/* Paranoia. */
8636	pMixedCtx->dr[7] &= UINT64_C(0xffffffff); /* upper 32 bits MBZ. */
8637	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8638	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8639
8640	rc \|= VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
8641	AssertRCReturn(rc,rc);
8642
8643	int rc2 = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8644	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8645	rc2 \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8646	AssertRCReturn(rc2, rc2);
8647	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8648	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8649	rc = VINF_SUCCESS;
8650	}
8651
8652	return rc;
8653	}
8654
8655
8656	/**
8657	* VM-exit exception handler for #NM (Device-not-available exception: floating
8658	* point exception).
8659	*/
8660	static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8661	{
8662	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8663
8664	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8665	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8666	#endif
8667
8668	/* We require CR0 and EFER. EFER is always up-to-date. */
8669	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8670	AssertRCReturn(rc, rc);
8671
8672	/* Lazy FPU loading; load the guest-FPU state transparently and continue execution of the guest. */
8673	PVM pVM = pVCpu->CTX_SUFF(pVM);
8674	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8675	if (rc == VINF_SUCCESS)
8676	{
8677	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8678	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8679	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8680	return VINF_SUCCESS;
8681	}
8682
8683	/* Forward #NM to the guest. */
8684	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8685	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8686	AssertRCReturn(rc, rc);
8687	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8688	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8690	return rc;
8691	}
8692
8693
8694	/**
8695	* VM-exit exception handler for #GP (General-protection exception).
8696	*
8697	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8698	*/
8699	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8700	{
8701	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8702	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
8703
8704	int rc = VERR_INTERNAL_ERROR_5;
8705	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8706	{
8707	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
8708	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8709	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8710	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8711	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8712	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8713	AssertRCReturn(rc, rc);
8714	Log(("#GP Gst: RIP %#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u\n", pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode,
8715	pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu)));
8716	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8717	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8718	return rc;
8719	#else
8720	/* We don't intercept #GP. */
8721	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8722	return VERR_VMX_UNEXPECTED_EXCEPTION;
8723	#endif
8724	}
8725
8726	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8727	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8728
8729	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8730	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8731	AssertRCReturn(rc, rc);
8732
8733	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8734	uint32_t cbOp = 0;
8735	PVM pVM = pVCpu->CTX_SUFF(pVM);
8736	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8737	if (RT_SUCCESS(rc))
8738	{
8739	rc = VINF_SUCCESS;
8740	Assert(cbOp == pDis->cbInstr);
8741	Log(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8742	switch (pDis->pCurInstr->uOpcode)
8743	{
8744	case OP_CLI:
8745	pMixedCtx->eflags.Bits.u1IF = 0;
8746	pMixedCtx->rip += pDis->cbInstr;
8747	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8748	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8749	break;
8750
8751	case OP_STI:
8752	pMixedCtx->eflags.Bits.u1IF = 1;
8753	pMixedCtx->rip += pDis->cbInstr;
8754	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8755	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8756	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8758	break;
8759
8760	case OP_HLT:
8761	rc = VINF_EM_HALT;
8762	pMixedCtx->rip += pDis->cbInstr;
8763	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8764	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8765	break;
8766
8767	case OP_POPF:
8768	{
8769	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8770	uint32_t cbParm = 0;
8771	uint32_t uMask = 0;
8772	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8773	{
8774	cbParm = 4;
8775	uMask = 0xffffffff;
8776	}
8777	else
8778	{
8779	cbParm = 2;
8780	uMask = 0xffff;
8781	}
8782
8783	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8784	RTGCPTR GCPtrStack = 0;
8785	X86EFLAGS uEflags;
8786	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8787	&GCPtrStack);
8788	if (RT_SUCCESS(rc))
8789	{
8790	Assert(sizeof(uEflags.u32) >= cbParm);
8791	uEflags.u32 = 0;
8792	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8793	}
8794	if (RT_FAILURE(rc))
8795	{
8796	rc = VERR_EM_INTERPRETER;
8797	break;
8798	}
8799	Log(("POPF %x -> %#RX64 mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8800	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8801	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8802	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8803	pMixedCtx->eflags.Bits.u1RF = 0;
8804	pMixedCtx->esp += cbParm;
8805	pMixedCtx->esp &= uMask;
8806	pMixedCtx->rip += pDis->cbInstr;
8807	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8808	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8809	break;
8810	}
8811
8812	case OP_PUSHF:
8813	{
8814	uint32_t cbParm = 0;
8815	uint32_t uMask = 0;
8816	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8817	{
8818	cbParm = 4;
8819	uMask = 0xffffffff;
8820	}
8821	else
8822	{
8823	cbParm = 2;
8824	uMask = 0xffff;
8825	}
8826
8827	/* Get the stack pointer & push the contents of eflags onto the stack. */
8828	RTGCPTR GCPtrStack = 0;
8829	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8830	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8831	if (RT_FAILURE(rc))
8832	{
8833	rc = VERR_EM_INTERPRETER;
8834	break;
8835	}
8836	X86EFLAGS uEflags;
8837	uEflags = pMixedCtx->eflags;
8838	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8839	uEflags.Bits.u1RF = 0;
8840	uEflags.Bits.u1VM = 0;
8841
8842	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8843	if (RT_FAILURE(rc))
8844	{
8845	rc = VERR_EM_INTERPRETER;
8846	break;
8847	}
8848	Log(("PUSHF %x -> %#RGv\n", uEflags.u, GCPtrStack));
8849	pMixedCtx->esp -= cbParm;
8850	pMixedCtx->esp &= uMask;
8851	pMixedCtx->rip += pDis->cbInstr;
8852	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8854	break;
8855	}
8856
8857	case OP_IRET:
8858	{
8859	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8860	* instruction reference. */
8861	RTGCPTR GCPtrStack = 0;
8862	uint32_t uMask = 0xffff;
8863	uint16_t aIretFrame[3];
8864	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8865	{
8866	rc = VERR_EM_INTERPRETER;
8867	break;
8868	}
8869	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8870	&GCPtrStack);
8871	if (RT_SUCCESS(rc))
8872	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8873	if (RT_FAILURE(rc))
8874	{
8875	rc = VERR_EM_INTERPRETER;
8876	break;
8877	}
8878	pMixedCtx->eip = 0;
8879	pMixedCtx->ip = aIretFrame[0];
8880	pMixedCtx->cs.Sel = aIretFrame[1];
8881	pMixedCtx->cs.ValidSel = aIretFrame[1];
8882	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8883	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8884	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8885	pMixedCtx->sp += sizeof(aIretFrame);
8886	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8887	\| HM_CHANGED_GUEST_RFLAGS;
8888	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8890	break;
8891	}
8892
8893	case OP_INT:
8894	{
8895	uint16_t uVector = pDis->Param1.uValue & 0xff;
8896	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8897	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8898	break;
8899	}
8900
8901	case OP_INTO:
8902	{
8903	if (pMixedCtx->eflags.Bits.u1OF)
8904	{
8905	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8907	}
8908	break;
8909	}
8910
8911	default:
8912	{
8913	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8914	EMCODETYPE_SUPERVISOR);
8915	rc = VBOXSTRICTRC_VAL(rc2);
8916	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8917	Log(("#GP rc=%Rrc\n", rc));
8918	break;
8919	}
8920	}
8921	}
8922	else
8923	rc = VERR_EM_INTERPRETER;
8924
8925	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8926	("#GP Unexpected rc=%Rrc\n", rc));
8927	return rc;
8928	}
8929
8930
8931	/**
8932	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8933	* the exception reported in the VMX transient structure back into the VM.
8934	*
8935	* @remarks Requires uExitIntrInfo in the VMX transient structure to be
8936	* up-to-date.
8937	*/
8938	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8939	{
8940	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8941
8942	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8943	hmR0VmxCheckExitDueToEventDelivery(). */
8944	int rc = hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8945	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8946	AssertRCReturn(rc, rc);
8947	Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
8948	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8949	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8950	return VINF_SUCCESS;
8951	}
8952
8953
8954	/**
8955	* VM-exit exception handler for #PF (Page-fault exception).
8956	*/
8957	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8958	{
8959	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8960	PVM pVM = pVCpu->CTX_SUFF(pVM);
8961	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8962	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8963	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8964	AssertRCReturn(rc, rc);
8965
8966	#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) \|\| defined(HMVMX_ALWAYS_TRAP_PF)
8967	if (pVM->hm.s.fNestedPaging)
8968	{
8969	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8970	{
8971	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
8972	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
8973	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8974	0 /* cbInstr */, pVmxTransient->uExitIntrErrorCode, pVmxTransient->uExitQualification);
8975	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8976	}
8977	else
8978	{
8979	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8980	pVCpu->hm.s.Event.fPending = false; /* A vectoring #PF. */
8981	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8982	Log(("Pending #DF due to vectoring #PF. NP\n"));
8983	}
8984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8985	return rc;
8986	}
8987	#else
8988	Assert(!pVM->hm.s.fNestedPaging);
8989	#endif
8990
8991	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8992	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8993	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8994	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8995	AssertRCReturn(rc, rc);
8996	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8997	if ( pVM->hm.s.fTRPPatchingAllowed
8998	&& pVM->hm.s.pGuestPatchMem
8999	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
9000	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
9001	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
9002	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
9003	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
9004	{
9005	RTGCPHYS GCPhys;
9006	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
9007	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
9008	if ( rc == VINF_SUCCESS
9009	&& GCPhys == GCPhysApicBase)
9010	{
9011	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9012	AssertRCReturn(rc, rc);
9013
9014	/* Only attempt to patch the instruction once. */
9015	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
9016	if (!pPatch)
9017	return VINF_EM_HM_PATCH_TPR_INSTR;
9018	}
9019	}
9020	#endif
9021
9022	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9023	AssertRCReturn(rc, rc);
9024
9025	Log(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
9026	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr3));
9027
9028	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
9029	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
9030	(RTGCPTR)pVmxTransient->uExitQualification);
9031
9032	Log(("#PF: rc=%Rrc\n", rc));
9033	if (rc == VINF_SUCCESS)
9034	{
9035	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
9036	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
9037	* memory? We don't update the whole state here... */
9038	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
9039	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
9040	TRPMResetTrap(pVCpu);
9041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
9042	return rc;
9043	}
9044	else if (rc == VINF_EM_RAW_GUEST_TRAP)
9045	{
9046	if (!pVmxTransient->fVectoringPF)
9047	{
9048	/* It's a guest page fault and needs to be reflected to the guest. */
9049	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
9050	TRPMResetTrap(pVCpu);
9051	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
9052	pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
9053	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
9054	0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
9055	}
9056	else
9057	{
9058	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
9059	TRPMResetTrap(pVCpu);
9060	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF for replace it with #DF. */
9061	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
9062	Log(("#PF: Pending #DF due to vectoring #PF\n"));
9063	}
9064
9065	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
9066	return VINF_SUCCESS;
9067	}
9068
9069	TRPMResetTrap(pVCpu);
9070	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
9071	return rc;
9072	}
9073

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 46214

Download in other formats: